diff --git a/changelog/unreleased/pull-2087 b/changelog/unreleased/pull-2087 new file mode 100644 index 000000000..36967c997 --- /dev/null +++ b/changelog/unreleased/pull-2087 @@ -0,0 +1,10 @@ +Enhancement: Add group-by option to snapshots command + +We have added an option to group the output of the snapshots command, similar +to the output of the forget command. The option has been called "--group-by" +and accepts any combination of the values "host", "paths" and "tags", separated +by commas. Default behavior (not specifying --group-by) has not been changed. +We have added support of the grouping to the JSON output. + +https://github.com/restic/restic/issues/2037 +https://github.com/restic/restic/pull/2087 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index a9b7246be..b047f1d45 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -4,10 +4,7 @@ import ( "context" "encoding/json" "io" - "sort" - "strings" - "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" "github.com/spf13/cobra" ) @@ -91,178 +88,129 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { return err } - // group by hostname and dirs - type key struct { - Hostname string - Paths []string - Tags []string - } - snapshotGroups := make(map[string]restic.Snapshots) - - var GroupByTag bool - var GroupByHost bool - var GroupByPath bool - var GroupOptionList []string - - GroupOptionList = strings.Split(opts.GroupBy, ",") - - for _, option := range GroupOptionList { - switch option { - case "host": - GroupByHost = true - case "paths": - GroupByPath = true - case "tags": - GroupByTag = true - case "": - default: - return errors.Fatal("unknown grouping option: '" + option + "'") - } - } - removeSnapshots := 0 ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() + + var snapshots restic.Snapshots + for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { - if len(args) > 0 { - // When explicit snapshots args are given, remove them immediately. + snapshots = append(snapshots, sn) + } + + if len(args) > 0 { + // When explicit snapshots args are given, remove them immediately. + for _, sn := range snapshots { if !opts.DryRun { h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} if err = repo.Backend().Remove(gopts.ctx, h); err != nil { return err } - Verbosef("removed snapshot %v\n", sn.ID().Str()) + if !gopts.JSON { + Verbosef("removed snapshot %v\n", sn.ID().Str()) + } removeSnapshots++ } else { - Verbosef("would have removed snapshot %v\n", sn.ID().Str()) + if !gopts.JSON { + Verbosef("would have removed snapshot %v\n", sn.ID().Str()) + } } - } else { - // Determining grouping-keys - var tags []string - var hostname string - var paths []string - - if GroupByTag { - tags = sn.Tags - sort.StringSlice(tags).Sort() - } - if GroupByHost { - hostname = sn.Hostname - } - if GroupByPath { - paths = sn.Paths - } - - sort.StringSlice(sn.Paths).Sort() - var k []byte - var err error - - k, err = json.Marshal(key{Tags: tags, Hostname: hostname, Paths: paths}) - - if err != nil { - return err - } - snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) } - } - - policy := restic.ExpirePolicy{ - Last: opts.Last, - Hourly: opts.Hourly, - Daily: opts.Daily, - Weekly: opts.Weekly, - Monthly: opts.Monthly, - Yearly: opts.Yearly, - Within: opts.Within, - Tags: opts.KeepTags, - } - - if policy.Empty() && len(args) == 0 { - Verbosef("no policy was specified, no snapshots will be removed\n") - } - - if !policy.Empty() { - if !gopts.JSON { - Verbosef("Applying Policy: %v\n", policy) + } else { + snapshotGroups, _, err := restic.GroupSnapshots(snapshots, opts.GroupBy) + if err != nil { + return err } - var jsonGroups []*ForgetGroup + policy := restic.ExpirePolicy{ + Last: opts.Last, + Hourly: opts.Hourly, + Daily: opts.Daily, + Weekly: opts.Weekly, + Monthly: opts.Monthly, + Yearly: opts.Yearly, + Within: opts.Within, + Tags: opts.KeepTags, + } - for k, snapshotGroup := range snapshotGroups { - var key key - if json.Unmarshal([]byte(k), &key) != nil { - return err - } - - var fg ForgetGroup - // Info + if policy.Empty() && len(args) == 0 { if !gopts.JSON { - Verbosef("snapshots") - } - var infoStrings []string - if GroupByTag { - infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") - fg.Tags = key.Tags - } - if GroupByHost { - infoStrings = append(infoStrings, "host ["+key.Hostname+"]") - fg.Host = key.Hostname - } - if GroupByPath { - infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") - fg.Paths = key.Paths - } - if infoStrings != nil && !gopts.JSON { - Verbosef(" for (" + strings.Join(infoStrings, ", ") + ")") + Verbosef("no policy was specified, no snapshots will be removed\n") } + } + + if !policy.Empty() { if !gopts.JSON { - Verbosef(":\n\n") + Verbosef("Applying Policy: %v\n", policy) } - keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + var jsonGroups []*ForgetGroup - if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("keep %d snapshots:\n", len(keep)) - PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Keep, keep) - - if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { - Printf("remove %d snapshots:\n", len(remove)) - PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) - Printf("\n") - } - addJSONSnapshots(&fg.Remove, remove) - - fg.Reasons = reasons - - jsonGroups = append(jsonGroups, &fg) - - removeSnapshots += len(remove) - - if !opts.DryRun { - for _, sn := range remove { - h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} - err = repo.Backend().Remove(gopts.ctx, h) + for k, snapshotGroup := range snapshotGroups { + if gopts.Verbose >= 1 && !gopts.JSON { + err = PrintSnapshotGroupHeader(gopts.stdout, k) if err != nil { return err } } - } - } - if gopts.JSON { - err = printJSONForget(gopts.stdout, jsonGroups) - if err != nil { - return err + var key restic.SnapshotGroupKey + if json.Unmarshal([]byte(k), &key) != nil { + return err + } + + var fg ForgetGroup + fg.Tags = key.Tags + fg.Host = key.Hostname + fg.Paths = key.Paths + + keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy) + + if len(keep) != 0 && !gopts.Quiet && !gopts.JSON { + Printf("keep %d snapshots:\n", len(keep)) + PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact) + Printf("\n") + } + addJSONSnapshots(&fg.Keep, keep) + + if len(remove) != 0 && !gopts.Quiet && !gopts.JSON { + Printf("remove %d snapshots:\n", len(remove)) + PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact) + Printf("\n") + } + addJSONSnapshots(&fg.Remove, remove) + + fg.Reasons = reasons + + jsonGroups = append(jsonGroups, &fg) + + removeSnapshots += len(remove) + + if !opts.DryRun { + for _, sn := range remove { + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + err = repo.Backend().Remove(gopts.ctx, h) + if err != nil { + return err + } + } + } + } + + if gopts.JSON { + err = printJSONForget(gopts.stdout, jsonGroups) + if err != nil { + return err + } } } } if removeSnapshots > 0 && opts.Prune { - Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots) + if !gopts.JSON { + Verbosef("%d snapshots have been removed, running prune\n", removeSnapshots) + } if !opts.DryRun { return pruneRepository(gopts, repo) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index d9623b942..94cd08836 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -32,6 +32,7 @@ type SnapshotOptions struct { Paths []string Compact bool Last bool + GroupBy string } var snapshotOptions SnapshotOptions @@ -45,6 +46,7 @@ func init() { f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)") f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format") f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path") + f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags") } func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error { @@ -64,25 +66,41 @@ func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) erro ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() - var list restic.Snapshots + var snapshots restic.Snapshots for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) { - list = append(list, sn) + snapshots = append(snapshots, sn) + } + snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy) + if err != nil { + return err } - if opts.Last { - list = FilterLastSnapshots(list) + for k, list := range snapshotGroups { + if opts.Last { + list = FilterLastSnapshots(list) + } + sort.Sort(sort.Reverse(list)) + snapshotGroups[k] = list } - sort.Sort(sort.Reverse(list)) - if gopts.JSON { - err := printSnapshotsJSON(gopts.stdout, list) + err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, grouped) if err != nil { - Warnf("error printing snapshot: %v\n", err) + Warnf("error printing snapshots: %v\n", err) } return nil } - PrintSnapshots(gopts.stdout, list, nil, opts.Compact) + + for k, list := range snapshotGroups { + if grouped { + err := PrintSnapshotGroupHeader(gopts.stdout, k) + if err != nil { + Warnf("error printing snapshots: %v\n", err) + return nil + } + } + PrintSnapshots(gopts.stdout, list, nil, opts.Compact) + } return nil } @@ -223,6 +241,42 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke tab.Write(stdout) } +// PrintSnapshotGroupHeader prints which group of the group-by option the +// following snapshots belong to. +// Prints nothing, if we did not group at all. +func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error { + var key restic.SnapshotGroupKey + var err error + + err = json.Unmarshal([]byte(groupKeyJSON), &key) + if err != nil { + return err + } + + if key.Hostname == "" && key.Tags == nil && key.Paths == nil { + return nil + } + + // Info + fmt.Fprintf(stdout, "snapshots") + var infoStrings []string + if key.Hostname != "" { + infoStrings = append(infoStrings, "host ["+key.Hostname+"]") + } + if key.Tags != nil { + infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]") + } + if key.Paths != nil { + infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]") + } + if infoStrings != nil { + fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", ")) + } + fmt.Fprintf(stdout, ":\n") + + return nil +} + // Snapshot helps to print Snaphots as JSON with their ID included. type Snapshot struct { *restic.Snapshot @@ -231,19 +285,58 @@ type Snapshot struct { ShortID string `json:"short_id"` } -// printSnapshotsJSON writes the JSON representation of list to stdout. -func printSnapshotsJSON(stdout io.Writer, list restic.Snapshots) error { +// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included. +type SnapshotGroup struct { + GroupKey restic.SnapshotGroupKey `json:"group_key"` + Snapshots []Snapshot `json:"snapshots"` +} +// printSnapshotsJSON writes the JSON representation of list to stdout. +func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error { + if grouped { + var snapshotGroups []SnapshotGroup + + for k, list := range snGroups { + var key restic.SnapshotGroupKey + var err error + var snapshots []Snapshot + + err = json.Unmarshal([]byte(k), &key) + if err != nil { + return err + } + + for _, sn := range list { + k := Snapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + } + snapshots = append(snapshots, k) + } + + group := SnapshotGroup{ + GroupKey: key, + Snapshots: snapshots, + } + snapshotGroups = append(snapshotGroups, group) + } + + return json.NewEncoder(stdout).Encode(snapshotGroups) + } + + // Old behavior var snapshots []Snapshot - for _, sn := range list { - - k := Snapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), + for _, list := range snGroups { + for _, sn := range list { + k := Snapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + } + snapshots = append(snapshots, k) } - snapshots = append(snapshots, k) } return json.NewEncoder(stdout).Encode(snapshots) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 6890e32aa..850e29579 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -56,6 +56,31 @@ Or filter by host: Combining filters is also possible. +Furthermore you can group the output by the same filters (host, paths, tags): + +.. code-block:: console + + $ restic -r /srv/restic-repo snapshots --group-by host + + enter password for repository: + snapshots for (host [kasimir]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + 40dc1520 2015-05-08 21:38:30 kasimir /home/user/work + 79766175 2015-05-08 21:40:19 kasimir /home/user/work + 2 snapshots + snapshots for (host [luigi]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + bdbd3439 2015-05-08 21:45:17 luigi /home/art + 9f0bc19e 2015-05-08 21:46:11 luigi /srv + 2 snapshots + snapshots for (host [kazik]) + ID Date Host Tags Directory + ---------------------------------------------------------------------- + 590c8fc8 2015-05-08 21:47:38 kazik /srv + 1 snapshots + Checking a repo's integrity and consistency =========================================== diff --git a/internal/restic/snapshot_group.go b/internal/restic/snapshot_group.go new file mode 100644 index 000000000..f18e0d767 --- /dev/null +++ b/internal/restic/snapshot_group.go @@ -0,0 +1,76 @@ +package restic + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/restic/restic/internal/errors" +) + +// SnapshotGroupKey is the structure for identifying groups in a grouped +// snapshot list. This is used by GroupSnapshots() +type SnapshotGroupKey struct { + Hostname string `json:"hostname"` + Paths []string `json:"paths"` + Tags []string `json:"tags"` +} + +// GroupSnapshots takes a list of snapshots and a grouping criteria and creates +// a group list of snapshots. +func GroupSnapshots(snapshots Snapshots, options string) (map[string]Snapshots, bool, error) { + // group by hostname and dirs + snapshotGroups := make(map[string]Snapshots) + + var GroupByTag bool + var GroupByHost bool + var GroupByPath bool + var GroupOptionList []string + + GroupOptionList = strings.Split(options, ",") + + for _, option := range GroupOptionList { + switch option { + case "host": + GroupByHost = true + case "paths": + GroupByPath = true + case "tags": + GroupByTag = true + case "": + default: + return nil, false, errors.Fatal("unknown grouping option: '" + option + "'") + } + } + + for _, sn := range snapshots { + // Determining grouping-keys + var tags []string + var hostname string + var paths []string + + if GroupByTag { + tags = sn.Tags + sort.StringSlice(tags).Sort() + } + if GroupByHost { + hostname = sn.Hostname + } + if GroupByPath { + paths = sn.Paths + } + + sort.StringSlice(sn.Paths).Sort() + var k []byte + var err error + + k, err = json.Marshal(SnapshotGroupKey{Tags: tags, Hostname: hostname, Paths: paths}) + + if err != nil { + return nil, false, err + } + snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn) + } + + return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil +}