diff --git a/doc/Design.md b/doc/Design.md index 117554d2b..52a228a93 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -285,7 +285,7 @@ This way, the password can be changed without having to re-encrypt all data. Snapshots --------- -A snapshots represents a directory with all files and sub-directories at a +A snapshot represents a directory with all files and sub-directories at a given point in time. For each backup that is made, a new snapshot is created. A snapshot is a JSON document that is stored in an encrypted file below the directory `snapshots` in the repository. The filename is the storage ID. This @@ -294,6 +294,31 @@ string is unique and used within restic to uniquely identify a snapshot. The command `restic cat snapshot` can be used as follows to decrypt and pretty-print the contents of a snapshot file: +```console +$ restic -r /tmp/restic-repo cat snapshot 251c2e58 +enter password for repository: +{ + "time": "2015-01-02T18:10:50.895208559+01:00", + "tree": "2da81727b6585232894cfbb8f8bdab8d1eccd3d8f7c92bc934d62e62e618ffdf", + "dir": "/tmp/testdata", + "hostname": "kasimir", + "username": "fd0", + "uid": 1000, + "gid": 100, + "tags": [ + "NL" + ] +} +``` + +Here it can be seen that this snapshot represents the contents of the directory +`/tmp/testdata`. The most important field is `tree`. When the meta data (e.g. +the tags) of a snapshot change, the snapshot needs to be re-encrypted and saved. +This will change the storage ID, so in order to relate these seemingly +different snapshots, a field `original` is introduced which contains the ID of +the original snapshot, e.g. after adding the tag `DE` to the snapshot above it +becomes: + ```console $ restic -r /tmp/restic-repo cat snapshot 22a5af1b enter password for repository: @@ -304,12 +329,17 @@ enter password for repository: "hostname": "kasimir", "username": "fd0", "uid": 1000, - "gid": 100 + "gid": 100, + "tags": [ + "NL", + "DE" + ], + "original": "251c2e5841355f743f9d4ffd3260bee765acee40a6229857e32b60446991b837" } ``` -Here it can be seen that this snapshot represents the contents of the directory -`/tmp/testdata`. The most important field is `tree`. +Once introduced, the `original` field is not modified when the snapshot's meta +data is changed again. All content within a restic repository is referenced according to its SHA-256 hash. Before saving, each file is split into variable sized Blobs of data. The diff --git a/doc/Manual.md b/doc/Manual.md index 3612b6f0e..513ef612c 100644 --- a/doc/Manual.md +++ b/doc/Manual.md @@ -73,6 +73,7 @@ Available Commands: rebuild-index build a new index file restore extract the data from a snapshot snapshots list all snapshots + tag modifies tags on snapshots unlock remove locks other processes created version Print version information @@ -394,6 +395,45 @@ enter password for repository: *eb78040b username kasimir 2015-08-12 13:29:57 ``` +# Manage tags + +Managing tags on snapshots is done with the `tag` command. The existing set of +tags can be replaced completely, tags can be added to removed. The result is +directly visible in the `snapshots` command. + +Let's say we want to tag snapshot `590c8fc8` with the tags `NL` and `CH` and +remove all other tags that may be present, the following command does that: + +```console +$ restic -r /tmp/backup tag --set NL,CH 590c8fc8 +Create exclusive lock for repository +Modified tags on 1 snapshots +``` + +Note the snapshot ID has changed, so between each change we need to look up +the new ID of the snapshot. But there is an even better way, the `tag` command +accepts `--tag` for a filter, so we can filter snapshots based on the tag we +just added. + +So we can add and remove tags incrementally like this: + +```console +$ restic -r /tmp/backup tag --tag NL --remove CH +Create exclusive lock for repository +Modified tags on 1 snapshots + +$ restic -r /tmp/backup tag --tag NL --add UK +Create exclusive lock for repository +Modified tags on 1 snapshots + +$ restic -r /tmp/backup tag --tag NL --remove NL +Create exclusive lock for repository +Modified tags on 1 snapshots + +$ restic -r /tmp/backup tag --tag NL --add SOMETHING +No snapshots were modified +``` + # Check integrity and consistency Imagine your repository is saved on a server that has a faulty hard drive, or diff --git a/src/cmds/restic/cmd_snapshots.go b/src/cmds/restic/cmd_snapshots.go index 634cab0cf..5e3db0c5c 100644 --- a/src/cmds/restic/cmd_snapshots.go +++ b/src/cmds/restic/cmd_snapshots.go @@ -166,7 +166,7 @@ func printSnapshotsReadable(stdout io.Writer, list []*restic.Snapshot) { type Snapshot struct { *restic.Snapshot - ID string `json:"id"` + ID *restic.ID `json:"id"` } // printSnapshotsJSON writes the JSON representation of list to stdout. @@ -178,7 +178,7 @@ func printSnapshotsJSON(stdout io.Writer, list []*restic.Snapshot) error { k := Snapshot{ Snapshot: sn, - ID: sn.ID().String(), + ID: sn.ID(), } snapshots = append(snapshots, k) } diff --git a/src/cmds/restic/cmd_tag.go b/src/cmds/restic/cmd_tag.go new file mode 100644 index 000000000..257320352 --- /dev/null +++ b/src/cmds/restic/cmd_tag.go @@ -0,0 +1,172 @@ +package main + +import ( + "github.com/spf13/cobra" + + "restic" + "restic/debug" + "restic/errors" + "restic/repository" +) + +var cmdTag = &cobra.Command{ + Use: "tag [flags] [snapshot-ID ...]", + Short: "modifies tags on snapshots", + Long: ` +The "tag" command allows you to modify tags on exiting snapshots. + +You can either set/replace the entire set of tags on a snapshot, or +add tags to/remove tags from the existing set. + +When no snapshot-ID is given, all snapshots matching the host, tag and path filter criteria are modified. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runTag(tagOptions, globalOptions, args) + }, +} + +// TagOptions bundles all options for the 'tag' command. +type TagOptions struct { + Host string + Paths []string + Tags []string + SetTags []string + AddTags []string + RemoveTags []string +} + +var tagOptions TagOptions + +func init() { + cmdRoot.AddCommand(cmdTag) + + tagFlags := cmdTag.Flags() + tagFlags.StringSliceVar(&tagOptions.SetTags, "set", nil, "`tag` which will replace the existing tags (can be given multiple times)") + tagFlags.StringSliceVar(&tagOptions.AddTags, "add", nil, "`tag` which will be added to the existing tags (can be given multiple times)") + tagFlags.StringSliceVar(&tagOptions.RemoveTags, "remove", nil, "`tag` which will be removed from the existing tags (can be given multiple times)") + + tagFlags.StringVarP(&tagOptions.Host, "host", "H", "", `only consider snapshots for this host, when no snapshot ID is given`) + tagFlags.StringSliceVar(&tagOptions.Tags, "tag", nil, "only consider snapshots which include this `tag`, when no snapshot-ID is given") + tagFlags.StringSliceVar(&tagOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given") +} + +func changeTags(repo *repository.Repository, snapshotID restic.ID, setTags, addTags, removeTags, tags, paths []string, host string) (bool, error) { + var changed bool + + sn, err := restic.LoadSnapshot(repo, snapshotID) + if err != nil { + return false, err + } + if (host != "" && host != sn.Hostname) || !sn.HasTags(tags) || !restic.SamePaths(sn.Paths, paths) { + return false, nil + } + + if len(setTags) != 0 { + // Setting the tag to an empty string really means no tags. + if len(setTags) == 1 && setTags[0] == "" { + setTags = nil + } + sn.Tags = setTags + changed = true + } else { + changed = sn.AddTags(addTags) + if sn.RemoveTags(removeTags) { + changed = true + } + } + + if changed { + // Retain the original snapshot id over all tag changes. + if sn.Original == nil { + sn.Original = sn.ID() + } + + // Save the new snapshot. + id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn) + if err != nil { + return false, err + } + + debug.Log("new snapshot saved as %v", id.Str()) + + if err = repo.Flush(); err != nil { + return false, err + } + + // Remove the old snapshot. + h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()} + if err = repo.Backend().Remove(h); err != nil { + return false, err + } + + debug.Log("old snapshot %v removed", sn.ID()) + } + return changed, nil +} + +func runTag(opts TagOptions, gopts GlobalOptions, args []string) error { + if len(opts.SetTags) == 0 && len(opts.AddTags) == 0 && len(opts.RemoveTags) == 0 { + return errors.Fatal("nothing to do!") + } + if len(opts.SetTags) != 0 && (len(opts.AddTags) != 0 || len(opts.RemoveTags) != 0) { + return errors.Fatal("--set and --add/--remove cannot be given at the same time") + } + + repo, err := OpenRepository(gopts) + if err != nil { + return err + } + + if !gopts.NoLock { + Verbosef("Create exclusive lock for repository\n") + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + var ids restic.IDs + if len(args) != 0 { + // When explit snapshot-IDs are given, the filtering does not matter anymore. + opts.Host = "" + opts.Tags = nil + opts.Paths = nil + + // Process all snapshot IDs given as arguments. + for _, s := range args { + snapshotID, err := restic.FindSnapshot(repo, s) + if err != nil { + Warnf("could not find a snapshot for ID %q, ignoring: %v\n", s, err) + continue + } + ids = append(ids, snapshotID) + } + ids = ids.Uniq() + } else { + // If there were no arguments, just get all snapshots. + done := make(chan struct{}) + defer close(done) + for snapshotID := range repo.List(restic.SnapshotFile, done) { + ids = append(ids, snapshotID) + } + } + + changeCnt := 0 + for _, id := range ids { + changed, err := changeTags(repo, id, opts.SetTags, opts.AddTags, opts.RemoveTags, opts.Tags, opts.Paths, opts.Host) + if err != nil { + Warnf("unable to modify the tags for snapshot ID %q, ignoring: %v\n", id, err) + continue + } + if changed { + changeCnt++ + } + } + if changeCnt == 0 { + Verbosef("No snapshots were modified\n") + } else { + Verbosef("Modified tags on %v snapshots\n", changeCnt) + } + return nil +} diff --git a/src/cmds/restic/integration_test.go b/src/cmds/restic/integration_test.go index 18c103cf1..9d7ea55fe 100644 --- a/src/cmds/restic/integration_test.go +++ b/src/cmds/restic/integration_test.go @@ -161,7 +161,7 @@ func testRunFind(t testing.TB, gopts GlobalOptions, pattern string) []string { return strings.Split(string(buf.Bytes()), "\n") } -func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string]Snapshot) { +func testRunSnapshots(t testing.TB, gopts GlobalOptions) (newest *Snapshot, snapmap map[restic.ID]Snapshot) { buf := bytes.NewBuffer(nil) globalOptions.stdout = buf globalOptions.JSON = true @@ -177,15 +177,14 @@ func testRunSnapshots(t testing.TB, gopts GlobalOptions) (*Snapshot, map[string] snapshots := []Snapshot{} OK(t, json.Unmarshal(buf.Bytes(), &snapshots)) - var newest *Snapshot - snapmap := make(map[string]Snapshot, len(snapshots)) + snapmap = make(map[restic.ID]Snapshot, len(snapshots)) for _, sn := range snapshots { - snapmap[sn.ID] = sn + snapmap[*sn.ID] = sn if newest == nil || sn.Time.After(newest.Time) { newest = &sn } } - return newest, snapmap + return } func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { @@ -655,6 +654,80 @@ func TestBackupTags(t *testing.T) { }) } +func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) { + OK(t, runTag(opts, gopts, []string{})) +} + +func TestTag(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) { + datafile := filepath.Join("testdata", "backup-data.tar.gz") + testRunInit(t, gopts) + SetupTarTestFixture(t, env.testdata, datafile) + + testRunBackup(t, []string{env.testdata}, BackupOptions{}, gopts) + testRunCheck(t, gopts) + newest, _ := testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 0, + "expected no tags, got %v", newest.Tags) + Assert(t, newest.Original == nil, + "expected original ID to be nil, got %v", newest.Original) + originalID := *newest.ID + + testRunTag(t, TagOptions{SetTags: []string{"NL"}}, gopts) + testRunCheck(t, gopts) + newest, _ = testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "NL", + "set failed, expected one NL tag, got %v", newest.Tags) + Assert(t, newest.Original != nil, "expected original snapshot id, got nil") + Assert(t, *newest.Original == originalID, + "expected original ID to be set to the first snapshot id") + + testRunTag(t, TagOptions{AddTags: []string{"CH"}}, gopts) + testRunCheck(t, gopts) + newest, _ = testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 2 && newest.Tags[0] == "NL" && newest.Tags[1] == "CH", + "add failed, expected CH,NL tags, got %v", newest.Tags) + Assert(t, newest.Original != nil, "expected original snapshot id, got nil") + Assert(t, *newest.Original == originalID, + "expected original ID to be set to the first snapshot id") + + testRunTag(t, TagOptions{RemoveTags: []string{"NL"}}, gopts) + testRunCheck(t, gopts) + newest, _ = testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 1 && newest.Tags[0] == "CH", + "remove failed, expected one CH tag, got %v", newest.Tags) + Assert(t, newest.Original != nil, "expected original snapshot id, got nil") + Assert(t, *newest.Original == originalID, + "expected original ID to be set to the first snapshot id") + + testRunTag(t, TagOptions{AddTags: []string{"US", "RU"}}, gopts) + testRunTag(t, TagOptions{RemoveTags: []string{"CH", "US", "RU"}}, gopts) + testRunCheck(t, gopts) + newest, _ = testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 0, + "expected no tags, got %v", newest.Tags) + Assert(t, newest.Original != nil, "expected original snapshot id, got nil") + Assert(t, *newest.Original == originalID, + "expected original ID to be set to the first snapshot id") + + // Check special case of removing all tags. + testRunTag(t, TagOptions{SetTags: []string{""}}, gopts) + testRunCheck(t, gopts) + newest, _ = testRunSnapshots(t, gopts) + Assert(t, newest != nil, "expected a new backup, got nil") + Assert(t, len(newest.Tags) == 0, + "expected no tags, got %v", newest.Tags) + Assert(t, newest.Original != nil, "expected original snapshot id, got nil") + Assert(t, *newest.Original == originalID, + "expected original ID to be set to the first snapshot id") + }) +} + func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { buf := bytes.NewBuffer(nil) diff --git a/src/restic/snapshot.go b/src/restic/snapshot.go index 91ffbd558..68f5e6878 100644 --- a/src/restic/snapshot.go +++ b/src/restic/snapshot.go @@ -21,6 +21,7 @@ type Snapshot struct { GID uint32 `json:"gid,omitempty"` Excludes []string `json:"excludes,omitempty"` Tags []string `json:"tags,omitempty"` + Original *ID `json:"original,omitempty"` id *ID // plaintext ID, used during restore } @@ -73,8 +74,7 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) { snapshots = append(snapshots, sn) } - - return snapshots, nil + return } func (sn Snapshot) String() string { @@ -99,6 +99,41 @@ func (sn *Snapshot) fillUserInfo() error { return err } +// AddTags adds the given tags to the snapshots tags, preventing duplicates. +// It returns true if any changes were made. +func (sn *Snapshot) AddTags(addTags []string) (changed bool) { +nextTag: + for _, add := range addTags { + for _, tag := range sn.Tags { + if tag == add { + continue nextTag + } + } + sn.Tags = append(sn.Tags, add) + changed = true + } + return +} + +// RemoveTags removes the given tags from the snapshots tags and +// returns true if any changes were made. +func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) { + for _, remove := range removeTags { + for i, tag := range sn.Tags { + if tag == remove { + // https://github.com/golang/go/wiki/SliceTricks + sn.Tags[i] = sn.Tags[len(sn.Tags)-1] + sn.Tags[len(sn.Tags)-1] = "" + sn.Tags = sn.Tags[:len(sn.Tags)-1] + + changed = true + break + } + } + } + return +} + // HasTags returns true if the snapshot has all the tags. func (sn *Snapshot) HasTags(tags []string) bool { nextTag: