Merge pull request #3300 from aawsome/backup-dryrun

backup: add --dry-run/-n flag

changelog/unreleased/issue-1542

@@ -0,0 +1,17 @@
+Enhancement: Add --dry-run/-n option to backup command
+
+Testing exclude filters and other configuration options required running a
+normal backup. Wrong filters could then cause files to be uploaded
+unexpectedly. It was also not possible to approximately determine beforehand
+how much data has to be uploaded.
+
+We added a new --dry-run/-n option to the backup command, which performs
+all the normal steps of a backup without actually writing any changes to
+the repository. Passing -vv will log information about files that would
+be added, allowing verification of source and exclusion backup options
+without committing changes to the repository.
+
+https://github.com/restic/restic/issues/1542
+https://github.com/restic/restic/pull/2308
+https://github.com/restic/restic/pull/3210
+https://github.com/restic/restic/pull/3300

cmd/restic/cmd_backup.go

@@ -92,6 +92,7 @@ type BackupOptions struct {
 	IgnoreInode             bool
 	IgnoreCtime             bool
 	UseFsSnapshot           bool
+	DryRun                  bool
 }
 
 var backupOptions BackupOptions
@@ -132,6 +133,7 @@ func init() {
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
 	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
+	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
 	}
@@ -535,6 +537,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		Run(ctx context.Context) error
 		Error(item string, fi os.FileInfo, err error) error
 		Finish(snapshotID restic.ID)
+		SetDryRun()
 
 		// ui.StdioWrapper
 		Stdout() io.WriteCloser
@@ -554,6 +557,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 		p = ui.NewBackup(term, gopts.verbosity)
 	}
 
+	if opts.DryRun {
+		repo.SetDryRun()
+		p.SetDryRun()
+	}
+
 	// use the terminal for stdout/stderr
 	prevStdout, prevStderr := gopts.stdout, gopts.stderr
 	defer func() {
@@ -722,7 +730,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
 
 	// Report finished execution
 	p.Finish(id)
-	if !gopts.JSON {
+	if !gopts.JSON && !opts.DryRun {
 		p.P("snapshot %s saved\n", id.Str())
 	}
 	if !success {

cmd/restic/integration_test.go

@@ -345,6 +345,57 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
 	testRunCheck(t, env.gopts)
 }
 
+func TestDryRunBackup(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{}
+	dryOpts := BackupOptions{DryRun: true}
+
+	// dry run before first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 0,
+		"expected no snapshot, got %v", snapshotIDs)
+	packIDs := testRunList(t, "packs", env.gopts)
+	rtest.Assert(t, len(packIDs) == 0,
+		"expected no data, got %v", snapshotIDs)
+	indexIDs := testRunList(t, "index", env.gopts)
+	rtest.Assert(t, len(indexIDs) == 0,
+		"expected no index, got %v", snapshotIDs)
+
+	// first backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// dry run between backups
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter := testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter := testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+
+	// second backup, implicit incremental
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	packIDs = testRunList(t, "packs", env.gopts)
+	indexIDs = testRunList(t, "index", env.gopts)
+
+	// another dry run
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
+	snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
+	rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
+	dataIDsAfter = testRunList(t, "packs", env.gopts)
+	rtest.Equals(t, packIDs, dataIDsAfter)
+	indexIDsAfter = testRunList(t, "index", env.gopts)
+	rtest.Equals(t, indexIDs, indexIDsAfter)
+}
+
 func TestBackupNonExistingFile(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()

doc/040_backup.rst

@@ -187,6 +187,23 @@ On **Windows**, a file is considered unchanged when its path, size
 and modification time match, and only ``--force`` has any effect.
 The other options are recognized but ignored.
 
+Dry Runs
+********
+
+You can perform a backup in dry run mode to see what would happen without
+modifying the repo.
+
+-  ``--dry-run``/``-n`` Report what would be done, without writing to the repository
+
+Combined with ``--verbose``, you can see a list of changes:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added"
+    modified  /plan.txt, saved in 0.000s (9.110 KiB added)
+    modified  /archive.tar.gz, saved in 0.140s (25.542 MiB added)
+    Would be added to the repo: 25.551 MiB
+
 Excluding Files
 ***************
 

internal/backend/dryrun/dry_backend.go

@@ -0,0 +1,79 @@
+package dryrun
+
+import (
+	"context"
+	"io"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+)
+
+// Backend passes reads through to an underlying layer and accepts writes, but
+// doesn't do anything. Also removes are ignored.
+// So in fact, this backend silently ignores all operations that would modify
+// the repo and does normal operations else.
+// This is used for `backup --dry-run`.
+type Backend struct {
+	b restic.Backend
+}
+
+// statically ensure that RetryBackend implements restic.Backend.
+var _ restic.Backend = &Backend{}
+
+// New returns a new backend that saves all data in a map in memory.
+func New(be restic.Backend) *Backend {
+	b := &Backend{b: be}
+	debug.Log("created new dry backend")
+	return b
+}
+
+// Save adds new Data to the backend.
+func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
+	if err := h.Valid(); err != nil {
+		return err
+	}
+
+	debug.Log("faked saving %v bytes at %v", rd.Length(), h)
+
+	// don't save anything, just return ok
+	return nil
+}
+
+// Remove deletes a file from the backend.
+func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
+	return nil
+}
+
+// Location returns the location of the backend.
+func (be *Backend) Location() string {
+	return "DRY:" + be.b.Location()
+}
+
+// Delete removes all data in the backend.
+func (be *Backend) Delete(ctx context.Context) error {
+	return nil
+}
+
+func (be *Backend) Close() error {
+	return be.b.Close()
+}
+
+func (be *Backend) IsNotExist(err error) bool {
+	return be.b.IsNotExist(err)
+}
+
+func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
+	return be.b.List(ctx, t, fn)
+}
+
+func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
+	return be.b.Load(ctx, h, length, offset, fn)
+}
+
+func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
+	return be.b.Stat(ctx, h)
+}
+
+func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
+	return be.b.Test(ctx, h)
+}

internal/backend/dryrun/dry_backend_test.go

@@ -0,0 +1,137 @@
+package dryrun_test
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+
+	"github.com/restic/restic/internal/backend/dryrun"
+	"github.com/restic/restic/internal/backend/mem"
+)
+
+// make sure that Backend implements backend.Backend
+var _ restic.Backend = &dryrun.Backend{}
+
+func newBackends() (*dryrun.Backend, restic.Backend) {
+	m := mem.New()
+	return dryrun.New(m), m
+}
+
+func TestDry(t *testing.T) {
+	ctx := context.TODO()
+
+	d, m := newBackends()
+	// Since the dry backend is a mostly write-only overlay, the standard backend test suite
+	// won't pass. Instead, perform a series of operations over the backend, testing the state
+	// at each step.
+	steps := []struct {
+		be      restic.Backend
+		op      string
+		fname   string
+		content string
+		wantErr string
+	}{
+		{d, "loc", "", "DRY:RAM", ""},
+		{d, "delete", "", "", ""},
+		{d, "stat", "a", "", "not found"},
+		{d, "list", "", "", ""},
+		{d, "save", "", "", "invalid"},
+		{d, "test", "a", "", ""},
+		{m, "save", "a", "baz", ""},  // save a directly to the mem backend
+		{d, "save", "b", "foob", ""}, // b is not saved
+		{d, "save", "b", "xxx", ""},  // no error as b is not saved
+		{d, "test", "a", "1", ""},
+		{d, "test", "b", "", ""},
+		{d, "stat", "", "", "invalid"},
+		{d, "stat", "a", "a 3", ""},
+		{d, "load", "a", "baz", ""},
+		{d, "load", "b", "", "not found"},
+		{d, "list", "", "a", ""},
+		{d, "remove", "c", "", ""},
+		{d, "stat", "b", "", "not found"},
+		{d, "list", "", "a", ""},
+		{d, "remove", "a", "", ""}, // a is in fact not removed
+		{d, "list", "", "a", ""},
+		{m, "remove", "a", "", ""}, // remove a from the mem backend
+		{d, "list", "", "", ""},
+		{d, "close", "", "", ""},
+		{d, "close", "", "", ""},
+	}
+
+	for i, step := range steps {
+		var err error
+		var boolRes bool
+
+		handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
+		switch step.op {
+		case "save":
+			err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content)))
+		case "test":
+			boolRes, err = step.be.Test(ctx, handle)
+			if boolRes != (step.content != "") {
+				t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "")
+			}
+		case "list":
+			fileList := []string{}
+			err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
+				fileList = append(fileList, fi.Name)
+				return nil
+			})
+			sort.Strings(fileList)
+			files := strings.Join(fileList, " ")
+			if files != step.content {
+				t.Errorf("%d. List = %q, want %q", i, files, step.content)
+			}
+		case "loc":
+			loc := step.be.Location()
+			if loc != step.content {
+				t.Errorf("%d. Location = %q, want %q", i, loc, step.content)
+			}
+		case "delete":
+			err = step.be.Delete(ctx)
+		case "remove":
+			err = step.be.Remove(ctx, handle)
+		case "stat":
+			var fi restic.FileInfo
+			fi, err = step.be.Stat(ctx, handle)
+			if err == nil {
+				fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)
+				if fis != step.content {
+					t.Errorf("%d. Stat = %q, want %q", i, fis, step.content)
+				}
+			}
+		case "load":
+			data := ""
+			err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error {
+				buf, err := ioutil.ReadAll(rd)
+				data = string(buf)
+				return err
+			})
+			if data != step.content {
+				t.Errorf("%d. Load = %q, want %q", i, data, step.content)
+			}
+		case "close":
+			err = step.be.Close()
+		default:
+			t.Fatalf("%d. unknown step operation %q", i, step.op)
+		}
+		if step.wantErr != "" {
+			if err == nil {
+				t.Errorf("%d. %s error = nil, want %q", i, step.op, step.wantErr)
+			} else if !strings.Contains(err.Error(), step.wantErr) {
+				t.Errorf("%d. %s error = %q, doesn't contain %q", i, step.op, err, step.wantErr)
+			} else if step.wantErr == "not found" && !step.be.IsNotExist(err) {
+				t.Errorf("%d. IsNotExist(%s error) = false, want true", i, step.op)
+			}
+
+		} else if err != nil {
+			t.Errorf("%d. %s error = %q, want nil", i, step.op, err)
+		}
+	}
+}

internal/repository/repository.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 
 	"github.com/restic/chunker"
+	"github.com/restic/restic/internal/backend/dryrun"
 	"github.com/restic/restic/internal/cache"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/debug"
@@ -72,6 +73,11 @@ func (r *Repository) UseCache(c *cache.Cache) {
 	r.be = c.Wrap(r.be)
 }
 
+// SetDryRun sets the repo backend into dry-run mode.
+func (r *Repository) SetDryRun() {
+	r.be = dryrun.New(r.be)
+}
+
 // PrefixLength returns the number of bytes required so that all prefixes of
 // all IDs of type t are unique.
 func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, error) {

internal/ui/backup.go

@@ -35,6 +35,7 @@ type Backup struct {
 	start time.Time
 
 	totalBytes uint64
+	dry        bool // true if writes are faked
 
 	totalCh     chan counter
 	processedCh chan counter
@@ -385,7 +386,11 @@ func (b *Backup) Finish(snapshotID restic.ID) {
 	b.P("Dirs:        %5d new, %5d changed, %5d unmodified\n", b.summary.Dirs.New, b.summary.Dirs.Changed, b.summary.Dirs.Unchanged)
 	b.V("Data Blobs:  %5d new\n", b.summary.ItemStats.DataBlobs)
 	b.V("Tree Blobs:  %5d new\n", b.summary.ItemStats.TreeBlobs)
-	b.P("Added to the repo: %-5s\n", formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize))
+	verb := "Added"
+	if b.dry {
+		verb = "Would add"
+	}
+	b.P("%s to the repo: %-5s\n", verb, formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize))
 	b.P("\n")
 	b.P("processed %v files, %v in %s",
 		b.summary.Files.New+b.summary.Files.Changed+b.summary.Files.Unchanged,
@@ -399,3 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) {
 func (b *Backup) SetMinUpdatePause(d time.Duration) {
 	b.MinUpdatePause = d
 }
+
+func (b *Backup) SetDryRun() {
+	b.dry = true
+}

internal/ui/json/backup.go

@@ -34,6 +34,7 @@ type Backup struct {
 	term  *termstatus.Terminal
 	v     uint
 	start time.Time
+	dry   bool
 
 	totalBytes uint64
 
@@ -403,6 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) {
 		TotalBytesProcessed: b.summary.ProcessedBytes,
 		TotalDuration:       time.Since(b.start).Seconds(),
 		SnapshotID:          snapshotID.Str(),
+		DryRun:              b.dry,
 	})
 }
 
@@ -412,6 +414,11 @@ func (b *Backup) SetMinUpdatePause(d time.Duration) {
 	b.MinUpdatePause = d
 }
 
+// SetDryRun marks the backup as a "dry run".
+func (b *Backup) SetDryRun() {
+	b.dry = true
+}
+
 type statusUpdate struct {
 	MessageType      string   `json:"message_type"` // "status"
 	SecondsElapsed   uint64   `json:"seconds_elapsed,omitempty"`
@@ -457,4 +464,5 @@ type summaryOutput struct {
 	TotalBytesProcessed uint64  `json:"total_bytes_processed"`
 	TotalDuration       float64 `json:"total_duration"` // in seconds
 	SnapshotID          string  `json:"snapshot_id"`
+	DryRun              bool    `json:"dry_run,omitempty"`
 }