restic/cmd/restic/cmd_restore_integration_test.go

package main

import (
	"context"
	"fmt"
	"io"
	"math/rand"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"testing"
	"time"

	"github.com/restic/restic/internal/feature"
	"github.com/restic/restic/internal/restic"
	rtest "github.com/restic/restic/internal/test"
	"github.com/restic/restic/internal/ui/termstatus"
)

func testRunRestore(t testing.TB, opts GlobalOptions, dir string, snapshotID restic.ID) {
	testRunRestoreExcludes(t, opts, dir, snapshotID, nil)
}

func testRunRestoreExcludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludes []string) {
	opts := RestoreOptions{
		Target: dir,
	}
	opts.Excludes = excludes

	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}

func testRunRestoreAssumeFailure(snapshotID string, opts RestoreOptions, gopts GlobalOptions) error {
	return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
		return runRestore(ctx, opts, gopts, term, []string{snapshotID})
	})
}

func testRunRestoreLatest(t testing.TB, gopts GlobalOptions, dir string, paths []string, hosts []string) {
	opts := RestoreOptions{
		Target: dir,
		SnapshotFilter: restic.SnapshotFilter{
			Hosts: hosts,
			Paths: paths,
		},
	}

	rtest.OK(t, testRunRestoreAssumeFailure("latest", opts, gopts))
}

func testRunRestoreIncludes(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includes []string) {
	opts := RestoreOptions{
		Target: dir,
	}
	opts.Includes = includes

	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}

func testRunRestoreIncludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, includesFile string) {
	opts := RestoreOptions{
		Target: dir,
	}
	opts.IncludeFiles = []string{includesFile}

	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}

func testRunRestoreExcludesFromFile(t testing.TB, gopts GlobalOptions, dir string, snapshotID restic.ID, excludesFile string) {
	opts := RestoreOptions{
		Target: dir,
	}
	opts.ExcludeFiles = []string{excludesFile}

	rtest.OK(t, testRunRestoreAssumeFailure(snapshotID.String(), opts, gopts))
}

func TestRestoreMustFailWhenUsingBothIncludesAndExcludes(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	// Add both include and exclude patterns
	includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
	excludePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}

	restoredir := filepath.Join(env.base, "restore")

	restoreOpts := RestoreOptions{
		Target: restoredir,
	}
	restoreOpts.Includes = includePatterns
	restoreOpts.Excludes = excludePatterns

	err := testRunRestoreAssumeFailure("latest", restoreOpts, env.gopts)
	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "exclude and include patterns are mutually exclusive"),
		"expected: %s error, got %v", "exclude and include patterns are mutually exclusive", err)
}

func TestRestoreIncludes(t *testing.T) {
	testfiles := []struct {
		path    string
		size    uint
		include bool // Whether this file should be included in the restore
	}{
		{"dir1/include_me.txt", 100, true},
		{"dir1/something_else.txt", 200, false},
		{"dir2/also_include_me.txt", 150, true},
		{"dir2/important_file.txt", 150, true},
		{"dir3/not_included.txt", 180, false},
		{"dir4/subdir/should_include_me.txt", 120, true},
	}

	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	// Create test files and directories
	for _, testFile := range testfiles {
		fullPath := filepath.Join(env.testdata, testFile.path)
		rtest.OK(t, os.MkdirAll(filepath.Dir(fullPath), 0755))
		rtest.OK(t, appendRandomData(fullPath, testFile.size))
	}

	opts := BackupOptions{}

	// Perform backup
	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	snapshotID := testListSnapshots(t, env.gopts, 1)[0]

	// Restore using includes
	includePatterns := []string{"dir1/*include_me.txt", "dir2/**", "dir4/**/*_me.txt"}
	restoredir := filepath.Join(env.base, "restore")
	testRunRestoreIncludes(t, env.gopts, restoredir, snapshotID, includePatterns)

	testRestoreFileInclusions := func(t *testing.T) {
		// Check that only the included files are restored
		for _, testFile := range testfiles {
			restoredFilePath := filepath.Join(restoredir, "testdata", testFile.path)
			_, err := os.Stat(restoredFilePath)
			if testFile.include {
				rtest.OK(t, err)
			} else {
				rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.path)
			}
		}
	}

	testRestoreFileInclusions(t)

	// Create an include file with some patterns
	patternsFile := env.base + "/patternsFile"
	fileErr := os.WriteFile(patternsFile, []byte(strings.Join(includePatterns, "\n")), 0644)
	if fileErr != nil {
		t.Fatalf("Could not write include file: %v", fileErr)
	}

	restoredir = filepath.Join(env.base, "restore-include-from-file")

	testRunRestoreIncludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)

	testRestoreFileInclusions(t)
}

func TestRestoreFilter(t *testing.T) {
	testfiles := []struct {
		name    string
		size    uint
		exclude bool
	}{
		{"testfile1.c", 100, true},
		{"testfile2.exe", 101, true},
		{"subdir1/subdir2/testfile3.docx", 102, true},
		{"subdir1/subdir2/testfile4.c", 102, false},
	}

	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	for _, testFile := range testfiles {
		p := filepath.Join(env.testdata, testFile.name)
		rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
		rtest.OK(t, appendRandomData(p, testFile.size))
	}

	opts := BackupOptions{}

	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	snapshotID := testListSnapshots(t, env.gopts, 1)[0]

	// no restore filter should restore all files
	testRunRestore(t, env.gopts, filepath.Join(env.base, "restore0"), snapshotID)
	for _, testFile := range testfiles {
		rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", testFile.name), int64(testFile.size)))
	}

	excludePatterns := []string{"testfile1.c", "*.exe", "*file3*"}

	// checks if the files are excluded correctly
	testRestoredFileExclusions := func(t *testing.T, restoredir string) {
		for _, testFile := range testfiles {
			restoredFilePath := filepath.Join(restoredir, "testdata", testFile.name)
			_, err := os.Stat(restoredFilePath)
			if testFile.exclude {
				rtest.Assert(t, os.IsNotExist(err), "File %s should not have been restored", testFile.name)
			} else {
				rtest.OK(t, testFileSize(restoredFilePath, int64(testFile.size)))
			}
		}
	}

	// restore with excludes
	restoredir := filepath.Join(env.base, "restore-with-excludes")
	testRunRestoreExcludes(t, env.gopts, restoredir, snapshotID, excludePatterns)
	testRestoredFileExclusions(t, restoredir)

	// Create an exclude file with some patterns
	patternsFile := env.base + "/patternsFile"
	fileErr := os.WriteFile(patternsFile, []byte(strings.Join(excludePatterns, "\n")), 0644)
	if fileErr != nil {
		t.Fatalf("Could not write include file: %v", fileErr)
	}

	// restore with excludes from file
	restoredir = filepath.Join(env.base, "restore-with-exclude-from-file")
	testRunRestoreExcludesFromFile(t, env.gopts, restoredir, snapshotID, patternsFile)

	testRestoredFileExclusions(t, restoredir)
}

func TestRestore(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	for i := 0; i < 10; i++ {
		p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
		rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
		rtest.OK(t, appendRandomData(p, uint(rand.Intn(2<<21))))
	}

	opts := BackupOptions{}

	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	// Restore latest without any filters
	restoredir := filepath.Join(env.base, "restore")
	testRunRestoreLatest(t, env.gopts, restoredir, nil, nil)

	diff := directoriesContentsDiff(env.testdata, filepath.Join(restoredir, filepath.Base(env.testdata)))
	rtest.Assert(t, diff == "", "directories are not equal %v", diff)
}

func TestRestoreLatest(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	p := filepath.Join(env.testdata, "testfile.c")
	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
	rtest.OK(t, appendRandomData(p, 100))

	opts := BackupOptions{}

	// chdir manually here so we can get the current directory. This is not the
	// same as the temp dir returned by os.MkdirTemp() on darwin.
	back := rtest.Chdir(t, filepath.Dir(env.testdata))
	defer back()

	curdir, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}

	testRunBackup(t, "", []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	rtest.OK(t, os.Remove(p))
	rtest.OK(t, appendRandomData(p, 101))
	testRunBackup(t, "", []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	// Restore latest without any filters
	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore0"), nil, nil)
	rtest.OK(t, testFileSize(filepath.Join(env.base, "restore0", "testdata", "testfile.c"), int64(101)))

	// Setup test files in different directories backed up in different snapshots
	p1 := filepath.Join(curdir, filepath.FromSlash("p1/testfile.c"))

	rtest.OK(t, os.MkdirAll(filepath.Dir(p1), 0755))
	rtest.OK(t, appendRandomData(p1, 102))
	testRunBackup(t, "", []string{"p1"}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	p2 := filepath.Join(curdir, filepath.FromSlash("p2/testfile.c"))

	rtest.OK(t, os.MkdirAll(filepath.Dir(p2), 0755))
	rtest.OK(t, appendRandomData(p2, 103))
	testRunBackup(t, "", []string{"p2"}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	p1rAbs := filepath.Join(env.base, "restore1", "p1/testfile.c")
	p2rAbs := filepath.Join(env.base, "restore2", "p2/testfile.c")

	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore1"), []string{filepath.Dir(p1)}, nil)
	rtest.OK(t, testFileSize(p1rAbs, int64(102)))
	if _, err := os.Stat(p2rAbs); os.IsNotExist(err) {
		rtest.Assert(t, os.IsNotExist(err),
			"expected %v to not exist in restore, but it exists, err %v", p2rAbs, err)
	}

	testRunRestoreLatest(t, env.gopts, filepath.Join(env.base, "restore2"), []string{filepath.Dir(p2)}, nil)
	rtest.OK(t, testFileSize(p2rAbs, int64(103)))
	if _, err := os.Stat(p1rAbs); os.IsNotExist(err) {
		rtest.Assert(t, os.IsNotExist(err),
			"expected %v to not exist in restore, but it exists, err %v", p1rAbs, err)
	}
}

func TestRestoreWithPermissionFailure(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	datafile := filepath.Join("testdata", "repo-restore-permissions-test.tar.gz")
	rtest.SetupTarTestFixture(t, env.base, datafile)

	snapshots := testListSnapshots(t, env.gopts, 1)

	_ = withRestoreGlobalOptions(func() error {
		globalOptions.stderr = io.Discard
		testRunRestore(t, env.gopts, filepath.Join(env.base, "restore"), snapshots[0])
		return nil
	})

	// make sure that all files have been restored, regardless of any
	// permission errors
	files := testRunLs(t, env.gopts, snapshots[0].String())
	for _, filename := range files {
		fi, err := os.Lstat(filepath.Join(env.base, "restore", filename))
		rtest.OK(t, err)

		rtest.Assert(t, !isFile(fi) || fi.Size() > 0,
			"file %v restored, but filesize is 0", filename)
	}
}

func setZeroModTime(filename string) error {
	var utimes = []syscall.Timespec{
		syscall.NsecToTimespec(0),
		syscall.NsecToTimespec(0),
	}

	return syscall.UtimesNano(filename, utimes)
}

func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	testRunInit(t, env.gopts)

	p := filepath.Join(env.testdata, "subdir1", "subdir2", "subdir3", "file.ext")
	rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
	rtest.OK(t, appendRandomData(p, 200))
	rtest.OK(t, setZeroModTime(filepath.Join(env.testdata, "subdir1", "subdir2")))

	opts := BackupOptions{}

	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
	testRunCheck(t, env.gopts)

	snapshotID := testListSnapshots(t, env.gopts, 1)[0]

	// restore with filter "*.ext", this should restore "file.ext", but
	// since the directories are ignored and only created because of
	// "file.ext", no meta data should be restored for them.
	testRunRestoreIncludes(t, env.gopts, filepath.Join(env.base, "restore0"), snapshotID, []string{"*.ext"})

	f1 := filepath.Join(env.base, "restore0", "testdata", "subdir1", "subdir2")
	_, err := os.Stat(f1)
	rtest.OK(t, err)

	// restore with filter "*", this should restore meta data on everything.
	testRunRestoreIncludes(t, env.gopts, filepath.Join(env.base, "restore1"), snapshotID, []string{"*"})

	f2 := filepath.Join(env.base, "restore1", "testdata", "subdir1", "subdir2")
	fi, err := os.Stat(f2)
	rtest.OK(t, err)

	rtest.Assert(t, fi.ModTime() == time.Unix(0, 0),
		"meta data of intermediate directory hasn't been restore")
}

func TestRestoreLocalLayout(t *testing.T) {
	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
	env, cleanup := withTestEnvironment(t)
	defer cleanup()

	var tests = []struct {
		filename string
		layout   string
	}{
		{"repo-layout-default.tar.gz", ""},
		{"repo-layout-s3legacy.tar.gz", ""},
		{"repo-layout-default.tar.gz", "default"},
		{"repo-layout-s3legacy.tar.gz", "s3legacy"},
	}

	for _, test := range tests {
		datafile := filepath.Join("..", "..", "internal", "backend", "testdata", test.filename)

		rtest.SetupTarTestFixture(t, env.base, datafile)

		env.gopts.extended["local.layout"] = test.layout

		// check the repo
		testRunCheck(t, env.gopts)

		// restore latest snapshot
		target := filepath.Join(env.base, "restore")
		testRunRestoreLatest(t, env.gopts, target, nil, nil)

		rtest.RemoveAll(t, filepath.Join(env.base, "repo"))
		rtest.RemoveAll(t, target)
	}
}