mirror
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #3592 from vgt/jsondiff 

Add json output for diff command
This commit is contained in:
MichaelEischer 2022-02-07 21:33:30 +01:00 committed by GitHub
parent 58236ead12 74f29ad09b
commit d8f58fb7bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 157 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
changelog/unreleased/issue-2508 Normal file
View File

@ -0,0 +1,11 @@
Enhancement: Support JSON output and quiet mode for diff
We've added support for getting machine-readable output for snapshot diff, just pass the
flag `--json` for `restic diff` and restic will output a JSON-encoded diff stats and change
list.
Passing the `--quiet` flag to the `diff` command will only print the summary
and suppress the detailed output.
https://github.com/restic/restic/issues/2508
https://github.com/restic/restic/pull/3592

111
cmd/restic/cmd_diff.go
View File

@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"path"
"reflect"
"sort"
@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
// Comparer collects all things needed to compare two snapshots.
type Comparer struct {
repo restic.Repository
opts DiffOptions
repo restic.Repository
opts DiffOptions
printChange func(change *Change)
}
type Change struct {
MessageType string `json:"message_type"` // "change"
Path string `json:"path"`
Modifier string `json:"modifier"`
}
func NewChange(path string, mode string) *Change {
return &Change{MessageType: "change", Path: path, Modifier: mode}
}
// DiffStat collects stats for all types of items.
type DiffStat struct {
Files, Dirs, Others int
DataBlobs, TreeBlobs int
Bytes uint64
Files int `json:"files"`
Dirs int `json:"dirs"`
Others int `json:"others"`
DataBlobs int `json:"data_blobs"`
TreeBlobs int `json:"tree_blobs"`
Bytes uint64 `json:"bytes"`
}
// Add adds stats information for node to s.
@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
}
}
// DiffStats collects the differences between two snapshots.
type DiffStats struct {
ChangedFiles int
Added DiffStat
Removed DiffStat
BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
}
// NewDiffStats creates new stats for a diff run.
func NewDiffStats() *DiffStats {
return &DiffStats{
BlobsBefore: restic.NewBlobSet(),
BlobsAfter: restic.NewBlobSet(),
BlobsCommon: restic.NewBlobSet(),
}
type DiffStatsContainer struct {
MessageType string `json:"message_type"` // "statistics"
SourceSnapshot string `json:"source_snapshot"`
TargetSnapshot string `json:"target_snapshot"`
ChangedFiles int `json:"changed_files"`
Added DiffStat `json:"added"`
Removed DiffStat `json:"removed"`
BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
}
// updateBlobs updates the blob counters in the stats struct.
@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
if node.Type == "dir" {
name += "/"
}
Printf("%-5s%v\n", mode, name)
c.printChange(NewChange(name, "+"))
stats.Add(node)
addBlobs(blobs, node)
@ -221,7 +229,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
return tree1Nodes, tree2Nodes, uniqueNames
}
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
debug.Log("diffing %v to %v", id1, id2)
tree1, err := c.repo.LoadTree(ctx, id1)
if err != nil {
@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
}
if mod != "" {
Printf("%-5s%v\n", mod, name)
c.printChange(NewChange(name, mod))
}
if node1.Type == "dir" && node2.Type == "dir" {
@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
if node1.Type == "dir" {
prefix += "/"
}
Printf("%-5s%v\n", "-", prefix)
c.printChange(NewChange(prefix, "-"))
stats.Removed.Add(node1)
if node1.Type == "dir" {
@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
if node2.Type == "dir" {
prefix += "/"
}
Printf("%-5s%v\n", "+", prefix)
c.printChange(NewChange(prefix, "+"))
stats.Added.Add(node2)
if node2.Type == "dir" {
@ -348,7 +356,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
return err
}
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
if !gopts.JSON {
Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
}
if sn1.Tree == nil {
return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@ -361,9 +371,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
c := &Comparer{
repo: repo,
opts: diffOptions,
printChange: func(change *Change) {
Printf("%-5s%v\n", change.Modifier, change.Path)
},
}
stats := NewDiffStats()
if gopts.JSON {
enc := json.NewEncoder(gopts.stdout)
c.printChange = func(change *Change) {
err := enc.Encode(change)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
}
}
if gopts.Quiet {
c.printChange = func(change *Change) {}
}
stats := &DiffStatsContainer{
MessageType: "statistics",
SourceSnapshot: args[0],
TargetSnapshot: args[1],
BlobsBefore: restic.NewBlobSet(),
BlobsAfter: restic.NewBlobSet(),
BlobsCommon: restic.NewBlobSet(),
}
stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
@ -376,14 +410,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
Printf("\n")
Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
if gopts.JSON {
err := json.NewEncoder(gopts.stdout).Encode(stats)
if err != nil {
Warnf("JSON encode failed: %v\n", err)
}
} else {
Printf("\n")
Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
}
return nil
}

76
cmd/restic/integration_test.go
View File

@ -159,8 +159,11 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
buf := bytes.NewBuffer(nil)
globalOptions.stdout = buf
oldStdout := gopts.stdout
gopts.stdout = buf
defer func() {
globalOptions.stdout = os.Stdout
gopts.stdout = oldStdout
}()
opts := DiffOptions{
@ -1972,10 +1975,8 @@ var diffOutputRegexPatterns = []string{
"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
}
func TestDiff(t *testing.T) {
func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testRunInit(t, env.gopts)
datadir := filepath.Join(env.base, "testdata")
@ -2011,19 +2012,82 @@ func TestDiff(t *testing.T) {
testRunBackup(t, "", []string{datadir}, opts, env.gopts)
_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
return env, cleanup, firstSnapshotID, secondSnapshotID
}
func TestDiff(t *testing.T) {
env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
defer cleanup()
// quiet suppresses the diff output except for the summary
env.gopts.Quiet = false
_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
if err != nil {
t.Fatalf("expected no error from diff for test repository, got %v", err)
}
rtest.OK(t, err)
for _, pattern := range diffOutputRegexPatterns {
r, err := regexp.Compile(pattern)
rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
}
// check quiet output
env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
}
type typeSniffer struct {
MessageType string `json:"message_type"`
}
func TestDiffJSON(t *testing.T) {
env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
defer cleanup()
// quiet suppresses the diff output except for the summary
env.gopts.Quiet = false
env.gopts.JSON = true
out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
var stat DiffStatsContainer
var changes int
scanner := bufio.NewScanner(strings.NewReader(out))
for scanner.Scan() {
line := scanner.Text()
var sniffer typeSniffer
rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
switch sniffer.MessageType {
case "change":
changes++
case "statistics":
rtest.OK(t, json.Unmarshal([]byte(line), &stat))
default:
t.Fatalf("unexpected message type %v", sniffer.MessageType)
}
}
rtest.Equals(t, 9, changes)
rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
stat.ChangedFiles == 1, "unexpected statistics")
// check quiet output
env.gopts.Quiet = true
outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
rtest.OK(t, err)
stat = DiffStatsContainer{}
rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
stat.ChangedFiles == 1, "unexpected statistics")
rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
}
type writeToOnly struct {