From 82ded35706521595e008650d9747c424d17dc0d5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 16 Nov 2017 21:51:12 +0100 Subject: [PATCH 1/4] node: Correctly compare times --- internal/restic/node.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/restic/node.go b/internal/restic/node.go index afa5d9792..81576057a 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -385,13 +385,13 @@ func (node Node) Equals(other Node) bool { if node.Mode != other.Mode { return false } - if node.ModTime != other.ModTime { + if !node.ModTime.Equal(other.ModTime) { return false } - if node.AccessTime != other.AccessTime { + if !node.AccessTime.Equal(other.AccessTime) { return false } - if node.ChangeTime != other.ChangeTime { + if !node.ChangeTime.Equal(other.ChangeTime) { return false } if node.UID != other.UID { From 2424012d7504e7f69884e54a69ec794806bf67a4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 15 Nov 2017 22:00:41 +0100 Subject: [PATCH 2/4] Add 'diff' command --- cmd/restic/cmd_diff.go | 340 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 cmd/restic/cmd_diff.go diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go new file mode 100644 index 000000000..d6eabd335 --- /dev/null +++ b/cmd/restic/cmd_diff.go @@ -0,0 +1,340 @@ +package main + +import ( + "context" + "path" + "reflect" + "sort" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" +) + +var cmdDiff = &cobra.Command{ + Use: "diff snapshot-ID snapshot-ID", + Short: "Show differences between two snapshots", + Long: ` +The "diff" command shows differences from the first to the second snapshot. The +first characters in each line display what has happened to a particular file or +directory: + + + The item was added + - The item was removed + M The metadata (access mode, timestamps, ...) for the item was changed + C The contents of a file has changed + T The type was changed, e.g. a file was made a symlink +`, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runDiff(diffOptions, globalOptions, args) + }, +} + +// DiffOptions collects all options for the diff command. +type DiffOptions struct { + ShowMetadata bool +} + +var diffOptions DiffOptions + +func init() { + cmdRoot.AddCommand(cmdDiff) + + f := cmdDiff.Flags() + f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") +} + +func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) { + id, err := restic.FindSnapshot(repo, desc) + if err != nil { + return nil, err + } + + return restic.LoadSnapshot(ctx, repo, id) +} + +// Comparer collects all things needed to compare two snapshots. +type Comparer struct { + repo restic.Repository + opts DiffOptions +} + +// DiffStats collects the differences between two snapshots. +type DiffStats struct { + FilesAdded, FilesRemoved, FilesChanged int + DirsAdded, DirsRemoved int + OthersAdded, OthersRemoved int + DataBlobsAdded, DataBlobsRemoved int + TreeBlobsAdded, TreeBlobsRemoved int + BytesAdded, BytesRemoved int + + blobsBefore, blobsAfter restic.BlobSet +} + +// NewDiffStats creates new stats for a diff run. +func NewDiffStats() *DiffStats { + return &DiffStats{ + blobsBefore: restic.NewBlobSet(), + blobsAfter: restic.NewBlobSet(), + } +} + +// AddNodeBefore records all blobs of node to the stats of the first snapshot. +func (stats *DiffStats) AddNodeBefore(node *restic.Node) { + if node == nil { + return + } + + switch node.Type { + case "file": + for _, blob := range node.Content { + h := restic.BlobHandle{ + ID: blob, + Type: restic.DataBlob, + } + stats.blobsBefore.Insert(h) + } + case "dir": + h := restic.BlobHandle{ + ID: *node.Subtree, + Type: restic.TreeBlob, + } + stats.blobsBefore.Insert(h) + } +} + +// AddNodeAfter records all blobs of node to the stats of the second snapshot. +func (stats *DiffStats) AddNodeAfter(node *restic.Node) { + if node == nil { + return + } + + switch node.Type { + case "file": + for _, blob := range node.Content { + h := restic.BlobHandle{ + ID: blob, + Type: restic.DataBlob, + } + stats.blobsAfter.Insert(h) + } + case "dir": + h := restic.BlobHandle{ + ID: *node.Subtree, + Type: restic.TreeBlob, + } + stats.blobsAfter.Insert(h) + } +} + +// UpdateBlobs updates the blob counters in the stats struct. +func (stats *DiffStats) UpdateBlobs(repo restic.Repository) { + both := stats.blobsBefore.Intersect(stats.blobsAfter) + for h := range stats.blobsBefore.Sub(both) { + switch h.Type { + case restic.DataBlob: + stats.DataBlobsRemoved++ + case restic.TreeBlob: + stats.TreeBlobsRemoved++ + } + + size, err := repo.LookupBlobSize(h.ID, h.Type) + if err != nil { + Warnf("unable to find blob size for %v: %v\n", h, err) + continue + } + + stats.BytesRemoved += int(size) + } + + for h := range stats.blobsAfter.Sub(both) { + switch h.Type { + case restic.DataBlob: + stats.DataBlobsAdded++ + case restic.TreeBlob: + stats.TreeBlobsAdded++ + } + + size, err := repo.LookupBlobSize(h.ID, h.Type) + if err != nil { + Warnf("unable to find blob size for %v: %v\n", h, err) + continue + } + + stats.BytesAdded += int(size) + } +} + +func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error { + debug.Log("diffing %v to %v", id1, id2) + tree1, err := c.repo.LoadTree(ctx, id1) + if err != nil { + return err + } + + tree2, err := c.repo.LoadTree(ctx, id2) + if err != nil { + return err + } + + uniqueNames := make(map[string]struct{}) + tree1Nodes := make(map[string]*restic.Node) + for _, node := range tree1.Nodes { + tree1Nodes[node.Name] = node + uniqueNames[node.Name] = struct{}{} + } + tree2Nodes := make(map[string]*restic.Node) + for _, node := range tree2.Nodes { + tree2Nodes[node.Name] = node + uniqueNames[node.Name] = struct{}{} + } + + names := make([]string, 0, len(uniqueNames)) + for name := range uniqueNames { + names = append(names, name) + } + + sort.Sort(sort.StringSlice(names)) + + for _, name := range names { + node1, t1 := tree1Nodes[name] + node2, t2 := tree2Nodes[name] + + stats.AddNodeBefore(node1) + stats.AddNodeAfter(node2) + + switch { + case t1 && t2: + name := path.Join(prefix, name) + mod := "" + + if node1.Type != node2.Type { + mod += "T" + } + + if node2.Type == "dir" { + name += "/" + } + + if node1.Type == "file" && + node2.Type == "file" && + !reflect.DeepEqual(node1.Content, node2.Content) { + mod += "C" + stats.FilesChanged++ + + if c.opts.ShowMetadata && !node1.Equals(*node2) { + mod += "M" + } + } else if c.opts.ShowMetadata && !node1.Equals(*node2) { + mod += "M" + } + + if mod != "" { + Printf(" % -3v %v\n", mod, name) + } + + if node1.Type == "dir" && node2.Type == "dir" { + err := c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree) + if err != nil { + Warnf("error: %v\n", err) + } + } + case t1 && !t2: + Printf("- %v\n", path.Join(prefix, name)) + switch node1.Type { + case "file": + stats.FilesRemoved++ + case "dir": + stats.DirsRemoved++ + default: + stats.OthersRemoved++ + } + case !t1 && t2: + Printf("+ %v\n", path.Join(prefix, name)) + switch node2.Type { + case "file": + stats.FilesAdded++ + case "dir": + stats.DirsAdded++ + default: + stats.OthersAdded++ + } + } + } + + return nil +} + +func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { + if len(args) != 2 { + return errors.Fatalf("specify two snapshot IDs") + } + + ctx, cancel := context.WithCancel(gopts.ctx) + defer cancel() + + repo, err := OpenRepository(gopts) + if err != nil { + return err + } + + if err = repo.LoadIndex(ctx); err != nil { + return err + } + + if !gopts.NoLock { + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + sn1, err := loadSnapshot(ctx, repo, args[0]) + if err != nil { + return err + } + + sn2, err := loadSnapshot(ctx, repo, args[1]) + if err != nil { + return err + } + + Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) + + if sn1.Tree == nil { + return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str()) + } + + if sn2.Tree == nil { + return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str()) + } + + c := &Comparer{ + repo: repo, + opts: diffOptions, + } + + stats := NewDiffStats() + + err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree) + if err != nil { + return err + } + + stats.UpdateBlobs(repo) + + Printf("\n") + Printf("Files: %5d new, %5d removed, %5d changed\n", stats.FilesAdded, stats.FilesRemoved, stats.FilesChanged) + Printf("Dirs: %5d new, %5d removed\n", stats.DirsAdded, stats.DirsRemoved) + Printf("Others: %5d new, %5d removed\n", stats.OthersAdded, stats.OthersRemoved) + Printf("Data Blobs: %5d new, %5d removed\n", stats.DataBlobsAdded, stats.DataBlobsRemoved) + Printf("Tree Blobs: %5d new, %5d removed\n", stats.TreeBlobsAdded, stats.TreeBlobsRemoved) + Printf(" Added: %-5s\n", formatBytes(uint64(stats.BytesAdded))) + Printf(" Removed: %-5s\n", formatBytes(uint64(stats.BytesRemoved))) + + return nil +} From 0722c44ba1e9243587a79e40c00b69b6ca5d6dc8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 Dec 2017 14:57:52 +0100 Subject: [PATCH 3/4] Add entry to CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78cf386e8..b32a5175d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ released version of restic from the perspective of the user. Important Changes in 0.X.Y ========================== + * The new command `diff` was added, it allows comparing two snapshots and + listing all differences. + https://github.com/restic/restic/issues/11 + https://github.com/restic/restic/issues/1460 + https://github.com/restic/restic/pull/1462 + Small changes ------------- From eb9e2bc79a5e786309a8ddd6601b668efe9a44d1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 Dec 2017 15:06:44 +0100 Subject: [PATCH 4/4] manual: Document `diff` command --- doc/040_backup.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 38e047984..034923cf1 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -127,6 +127,31 @@ args: $ restic -r /tmp/backup backup --files-from /tmp/files_to_backup /tmp/some_additional_file +Comparing Snapshots +******************* + +Restic has a `diff` command which shows the difference between two snapshots +and displays a small statistic, just pass the command two snapshot IDs: + +.. code-block:: console + + $ restic -r /tmp/backup diff 5845b002 2ab627a6 + password is correct + comparing snapshot ea657ce5 to 2ab627a6: + + C /restic/cmd_diff.go + + /restic/foo + C /restic/restic + + Files: 0 new, 0 removed, 2 changed + Dirs: 1 new, 0 removed + Others: 0 new, 0 removed + Data Blobs: 14 new, 15 removed + Tree Blobs: 2 new, 1 removed + Added: 16.403 MiB + Removed: 16.402 MiB + + Backing up special items and metadata *************************************