From 6e38a8a033a87e45f4beae6071302a0d1872264c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 17 May 2015 20:48:59 +0200 Subject: [PATCH 1/7] Move FindSnapshot, make Repository.List() return IDs --- backend/generic.go | 12 --------- cmd/restic/cmd_backup.go | 29 ++++++--------------- cmd/restic/cmd_cat.go | 7 +---- cmd/restic/cmd_find.go | 11 +++----- cmd/restic/cmd_fsck.go | 15 ++--------- cmd/restic/cmd_key.go | 10 ++++---- cmd/restic/cmd_ls.go | 7 +---- cmd/restic/cmd_restore.go | 8 +----- cmd/restic/cmd_snapshots.go | 8 +----- repository/key.go | 2 +- repository/repository.go | 51 +++++++++++++++++++++++++++++-------- snapshot.go | 12 +++++++++ 12 files changed, 75 insertions(+), 97 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index c7d569972..64e48ae68 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -56,18 +56,6 @@ func Find(be Lister, t Type, prefix string) (string, error) { return "", ErrNoIDPrefixFound } -// FindSnapshot takes a string and tries to find a snapshot whose ID matches -// the string as closely as possible. -func FindSnapshot(be Lister, s string) (string, error) { - // find snapshot id with prefix - name, err := Find(be, Snapshot, s) - if err != nil { - return "", err - } - - return name, nil -} - // PrefixLength returns the number of bytes required so that all prefixes of // all names of type t are unique. func PrefixLength(be Lister, t Type) (int, error) { diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f1e116265..08a5ff46d 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -185,28 +185,20 @@ func (cmd CmdBackup) Execute(args []string) error { return err } - var ( - parentSnapshot string - parentSnapshotID backend.ID - ) + var parentSnapshotID backend.ID // Force using a parent if !cmd.Force && cmd.Parent != "" { - parentSnapshot, err = s.FindSnapshot(cmd.Parent) + parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Parent, err) } - parentSnapshotID, err = backend.ParseID(parentSnapshot) - if err != nil { - return fmt.Errorf("invalid parent snapshot id %v", parentSnapshot) - } - fmt.Printf("found parent snapshot %v\n", parentSnapshotID) } // Find last snapshot to set it as parent, if not already set - if !cmd.Force && parentSnapshot == "" { + if !cmd.Force && parentSnapshotID == nil { samePaths := func(expected, actual []string) bool { if len(expected) != len(actual) { return false @@ -221,24 +213,19 @@ func (cmd CmdBackup) Execute(args []string) error { var latest time.Time - for snapshotIDString := range s.List(backend.Snapshot, make(chan struct{})) { - snapshotID, err := backend.ParseID(snapshotIDString) - if err != nil { - return fmt.Errorf("Error with the listing of snapshots inputting invalid backend ids: %v", err) - } - - snapshot, err := restic.LoadSnapshot(s, snapshotID) + for id := range s.List(backend.Snapshot, make(chan struct{})) { + snapshot, err := restic.LoadSnapshot(s, id) if err != nil { return fmt.Errorf("Error listing snapshot: %v", err) } if snapshot.Time.After(latest) && samePaths(snapshot.Paths, target) { latest = snapshot.Time - parentSnapshotID = snapshotID - parentSnapshot = snapshotIDString + parentSnapshotID = id } } - if parentSnapshot != "" { + + if parentSnapshotID != nil { fmt.Printf("using parent snapshot %v\n", parentSnapshotID) } } diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index e10a9071b..65e60c51f 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -53,12 +53,7 @@ func (cmd CmdCat) Execute(args []string) error { } // find snapshot id with prefix - name, err := s.FindSnapshot(args[1]) - if err != nil { - return err - } - - id, err = backend.ParseID(name) + id, err = restic.FindSnapshot(s, args[1]) if err != nil { return err } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 10c1de4bd..932d596cf 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -105,13 +105,8 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str return results, nil } -func (c CmdFind) findInSnapshot(repo *repository.Repository, name string) error { - debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", name, c.oldest, c.newest) - - id, err := backend.ParseID(name) - if err != nil { - return err - } +func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error { + debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest) sn, err := restic.LoadSnapshot(repo, id) if err != nil { @@ -169,7 +164,7 @@ func (c CmdFind) Execute(args []string) error { c.pattern = args[0] if c.Snapshot != "" { - snapshotID, err := backend.FindSnapshot(s, c.Snapshot) + snapshotID, err := restic.FindSnapshot(s, c.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", args[1], err) } diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index efd28956a..a9015a797 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -198,16 +198,11 @@ func (cmd CmdFsck) Execute(args []string) error { } if cmd.Snapshot != "" { - name, err := s.FindSnapshot(cmd.Snapshot) + id, err := restic.FindSnapshot(s, cmd.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err) } - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name) - } - err = fsckSnapshot(cmd, s, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) @@ -225,13 +220,7 @@ func (cmd CmdFsck) Execute(args []string) error { defer close(done) var firstErr error - for name := range s.List(backend.Snapshot, done) { - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name) - continue - } - + for id := range s.List(backend.Snapshot, done) { err = fsckSnapshot(cmd, s, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index d962fdffc..8ae2ac052 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -34,20 +34,20 @@ func listKeys(s *repository.Repository) error { done := make(chan struct{}) defer close(done) - for name := range s.List(backend.Key, done) { - k, err := repository.LoadKey(s, name) + for id := range s.List(backend.Key, done) { + k, err := repository.LoadKey(s, id.String()) if err != nil { fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err) continue } var current string - if name == s.KeyName() { + if id.String() == s.KeyName() { current = "*" } else { current = " " } - tab.Rows = append(tab.Rows, []interface{}{current, name[:plen], + tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen], k.Username, k.Hostname, k.Created.Format(TimeFormat)}) } @@ -133,7 +133,7 @@ func (cmd CmdKey) Execute(args []string) error { case "add": return addKey(s) case "rm": - id, err := backend.Find(s, backend.Key, args[1]) + id, err := backend.Find(s.Backend(), backend.Key, args[1]) if err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index ee07bcf0d..1f75b515c 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -77,12 +77,7 @@ func (cmd CmdLs) Execute(args []string) error { return err } - name, err := backend.FindSnapshot(s, args[0]) - if err != nil { - return err - } - - id, err := backend.ParseID(name) + id, err := restic.FindSnapshot(s, args[0]) if err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 5d6589a82..3d14b17fb 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -6,7 +6,6 @@ import ( "path/filepath" "github.com/restic/restic" - "github.com/restic/restic/backend" ) type CmdRestore struct{} @@ -40,12 +39,7 @@ func (cmd CmdRestore) Execute(args []string) error { return err } - name, err := backend.FindSnapshot(s, args[0]) - if err != nil { - errx(1, "invalid id %q: %v", args[0], err) - } - - id, err := backend.ParseID(name) + id, err := restic.FindSnapshot(s, args[0]) if err != nil { errx(1, "invalid id %q: %v", args[0], err) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index f5ebcd3fd..3703884b2 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -105,13 +105,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { defer close(done) list := []*restic.Snapshot{} - for name := range s.List(backend.Snapshot, done) { - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "error parsing id: %v", name) - continue - } - + for id := range s.List(backend.Snapshot, done) { sn, err := restic.LoadSnapshot(s, id) if err != nil { fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) diff --git a/repository/key.go b/repository/key.go index 8a2ec4530..ce8a661d6 100644 --- a/repository/key.go +++ b/repository/key.go @@ -98,7 +98,7 @@ func SearchKey(s *Repository, password string) (*Key, error) { // try all keys in repo done := make(chan struct{}) defer close(done) - for name := range s.List(backend.Key, done) { + for name := range s.Backend().List(backend.Key, done) { key, err := OpenKey(s, name, password) if err != nil { continue diff --git a/repository/repository.go b/repository/repository.go index 16c3d8a14..5dc273e6c 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -52,12 +52,6 @@ func (s *Repository) Find(t backend.Type, prefix string) (string, error) { return backend.Find(s.be, t, prefix) } -// FindSnapshot takes a string and tries to find a snapshot whose ID matches -// the string as closely as possible. -func (s *Repository) FindSnapshot(name string) (string, error) { - return backend.FindSnapshot(s.be, name) -} - // PrefixLength returns the number of bytes required so that all prefixes of // all IDs of type t are unique. func (s *Repository) PrefixLength(t backend.Type) (int, error) { @@ -637,14 +631,49 @@ func (s *Repository) Count(t backend.Type) (n uint) { return } -// Proxy methods to backend +func (s *Repository) list(t backend.Type, done <-chan struct{}, out chan<- backend.ID) { + defer close(out) + in := s.be.List(t, done) -func (s *Repository) Get(t backend.Type, name string) (io.ReadCloser, error) { - return s.be.Get(t, name) + var ( + // disable sending on the outCh until we received a job + outCh chan<- backend.ID + // enable receiving from in + inCh = in + id backend.ID + err error + ) + + for { + select { + case <-done: + return + case strID, ok := <-inCh: + if !ok { + // input channel closed, we're done + return + } + id, err = backend.ParseID(strID) + if err != nil { + // ignore invalid IDs + continue + } + + inCh = nil + outCh = out + case outCh <- id: + outCh = nil + inCh = in + } + } } -func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan string { - return s.be.List(t, done) +func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.ID { + outCh := make(chan backend.ID) + + go s.list(t, done, outCh) + + return outCh } func (s *Repository) Test(t backend.Type, name string) (bool, error) { diff --git a/snapshot.go b/snapshot.go index c5abca5e1..e308f0d04 100644 --- a/snapshot.go +++ b/snapshot.go @@ -89,3 +89,15 @@ func (sn *Snapshot) fillUserInfo() error { return nil } + +// FindSnapshot takes a string and tries to find a snapshot whose ID matches +// the string as closely as possible. +func FindSnapshot(repo *repository.Repository, s string) (backend.ID, error) { + // find snapshot id with prefix + name, err := backend.Find(repo.Backend(), backend.Snapshot, s) + if err != nil { + return nil, err + } + + return backend.ParseID(name) +} From eb6dfcf58cd0330ffae1934a8bed0f2d89493973 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 17 May 2015 20:51:32 +0200 Subject: [PATCH 2/7] Remove Repository.Test() --- archiver.go | 2 +- cache.go | 2 +- cmd/restic/cmd_fsck.go | 2 +- repository/repository.go | 6 +----- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/archiver.go b/archiver.go index 286cd0d8f..00d299f4d 100644 --- a/archiver.go +++ b/archiver.go @@ -254,7 +254,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan st // check if all content is still available in the repository contentMissing := false for _, blob := range oldNode.blobs { - if ok, err := arch.repo.Test(backend.Data, blob.Storage.String()); !ok || err != nil { + if ok, err := arch.repo.Backend().Test(backend.Data, blob.Storage.String()); !ok || err != nil { debug.Log("Archiver.fileWorker", " %v not using old data, %v (%v) is missing", e.Path(), blob.ID.Str(), blob.Storage.Str()) contentMissing = true break diff --git a/cache.go b/cache.go index 5e419bf2f..089c3f257 100644 --- a/cache.go +++ b/cache.go @@ -115,7 +115,7 @@ func (c *Cache) Clear(repo *repository.Repository) error { for _, entry := range list { debug.Log("Cache.Clear", "found entry %v", entry) - if ok, err := repo.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil { + if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil { debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry) err = c.purge(backend.Snapshot, entry.Subtype, entry.ID) diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index a9015a797..f79fb8e56 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -58,7 +58,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint } } else { // test if data blob is there - ok, err := repo.Test(backend.Data, packID.String()) + ok, err := repo.Backend().Test(backend.Data, packID.String()) if err != nil { return 0, err } diff --git a/repository/repository.go b/repository/repository.go index 5dc273e6c..c2adf0d66 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -580,7 +580,7 @@ func (s *Repository) SearchKey(password string) error { // Init creates a new master key with the supplied password and initializes the // repository config. func (s *Repository) Init(password string) error { - has, err := s.Test(backend.Config, "") + has, err := s.be.Test(backend.Config, "") if err != nil { return err } @@ -676,10 +676,6 @@ func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I return outCh } -func (s *Repository) Test(t backend.Type, name string) (bool, error) { - return s.be.Test(t, name) -} - func (s *Repository) Remove(t backend.Type, name string) error { return s.be.Remove(t, name) } From af381c2ab0fbd4ae63967dbdcd79f0a662325222 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 17 May 2015 20:52:31 +0200 Subject: [PATCH 3/7] Remove Repository.Remove() --- cmd/restic/cmd_key.go | 4 ++-- repository/repository.go | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 8ae2ac052..344d16750 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -79,7 +79,7 @@ func deleteKey(repo *repository.Repository, name string) error { return errors.New("refusing to remove key currently used to access repository") } - err := repo.Remove(backend.Key, name) + err := repo.Backend().Remove(backend.Key, name) if err != nil { return err } @@ -103,7 +103,7 @@ func changePassword(s *repository.Repository) error { } // remove old key - err = s.Remove(backend.Key, s.KeyName()) + err = s.Backend().Remove(backend.Key, s.KeyName()) if err != nil { return err } diff --git a/repository/repository.go b/repository/repository.go index c2adf0d66..8bc2b4992 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -676,10 +676,6 @@ func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I return outCh } -func (s *Repository) Remove(t backend.Type, name string) error { - return s.be.Remove(t, name) -} - func (s *Repository) Close() error { return s.be.Close() } From f773feeb0495e68c18c7b02bf614dcb82db7204a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 17 May 2015 20:55:24 +0200 Subject: [PATCH 4/7] Remove Repository.Close() and .Location() --- repository/repository.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index 8bc2b4992..aad5f3c9b 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -676,10 +676,6 @@ func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I return outCh } -func (s *Repository) Close() error { - return s.be.Close() -} - func (s *Repository) Delete() error { if b, ok := s.be.(backend.Deleter); ok { return b.Delete() @@ -688,6 +684,6 @@ func (s *Repository) Delete() error { return errors.New("Delete() called for backend that does not implement this method") } -func (s *Repository) Location() string { - return s.be.Location() +func (s *Repository) Close() error { + return s.be.Close() } From 65c3aead3ee2ef93864ab35bc67d25f5e5ea5826 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 16 May 2015 14:05:19 +0200 Subject: [PATCH 5/7] Add index.Dump() --- repository/index.go | 71 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/repository/index.go b/repository/index.go index be7904c66..29a725872 100644 --- a/repository/index.go +++ b/repository/index.go @@ -173,28 +173,24 @@ type blobJSON struct { Length uint `json:"length"` } -// Encode writes the JSON serialization of the index to the writer w. This -// serialization only contains new blobs added via idx.Store(), not old ones -// introduced via DecodeIndex(). -func (idx *Index) Encode(w io.Writer) error { - debug.Log("Index.Encode", "encoding index") - idx.m.Lock() - defer idx.m.Unlock() - +// generatePackList returns a list of packs containing only the index entries +// that selsectFn returned true for. If selectFn is nil, the list contains all +// blobs in the index. +func (idx *Index) generatePackList(selectFn func(indexEntry) bool) ([]*packJSON, error) { list := []*packJSON{} packs := make(map[string]*packJSON) for id, blob := range idx.pack { - if blob.old { + if selectFn != nil && !selectFn(blob) { continue } - debug.Log("Index.Encode", "handle blob %q", id[:8]) + debug.Log("Index.generatePackList", "handle blob %q", id[:8]) if blob.packID == nil { - debug.Log("Index.Encode", "blob %q has no packID! (type %v, offset %v, length %v)", + debug.Log("Index.generatePackList", "blob %q has no packID! (type %v, offset %v, length %v)", id[:8], blob.tpe, blob.offset, blob.length) - return fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id) + return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id) } // see if pack is already in map @@ -217,12 +213,63 @@ func (idx *Index) Encode(w io.Writer) error { }) } + debug.Log("Index.generatePackList", "done") + + return list, nil +} + +// encode writes the JSON serialization of the index filtered by selectFn to enc. +func (idx *Index) encode(w io.Writer, selectFn func(indexEntry) bool) error { + list, err := idx.generatePackList(func(entry indexEntry) bool { + return !entry.old + }) + if err != nil { + return err + } + debug.Log("Index.Encode", "done") enc := json.NewEncoder(w) return enc.Encode(list) } +// Encode writes the JSON serialization of the index to the writer w. This +// serialization only contains new blobs added via idx.Store(), not old ones +// introduced via DecodeIndex(). +func (idx *Index) Encode(w io.Writer) error { + debug.Log("Index.Encode", "encoding index") + idx.m.Lock() + defer idx.m.Unlock() + + return idx.encode(w, func(e indexEntry) bool { return !e.old }) +} + +// Dump writes the pretty-printed JSON representation of the index to w. +func (idx *Index) Dump(w io.Writer) error { + debug.Log("Index.Dump", "dumping index") + idx.m.Lock() + defer idx.m.Unlock() + + list, err := idx.generatePackList(nil) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(list, "", " ") + if err != nil { + return err + } + + _, err = w.Write(append(buf, '\n')) + if err != nil { + return err + } + + debug.Log("Index.Dump", "done") + + return nil +} + // DecodeIndex loads and unserializes an index from rd. func DecodeIndex(rd io.Reader) (*Index, error) { debug.Log("Index.DecodeIndex", "Start decoding index") From df1e3e03e923a5b66cb0e3373dfeae2c43f1e8a0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 16 May 2015 14:24:24 +0200 Subject: [PATCH 6/7] Add 'dump' command 'dump' writes internal data structes pretty-printed as JSON to stdout. This was done to debug fsck error messages. In contrast to the 'cat' command, this one prints the data structures as there are interpreted by restic, not as they are stored in the repository. This means that only the merged index from all the index files is printed out. This is meant for debugging only, it's compiled only when the "debug" tag is active. --- cmd/restic/cmd_dump.go | 149 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 cmd/restic/cmd_dump.go diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go new file mode 100644 index 000000000..7e7a54293 --- /dev/null +++ b/cmd/restic/cmd_dump.go @@ -0,0 +1,149 @@ +// +build debug + +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/juju/errors" + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/pack" + "github.com/restic/restic/repository" +) + +type CmdDump struct{} + +func init() { + _, err := parser.AddCommand("dump", + "dump data structures", + "The dump command dumps data structures from a repository as JSON documents", + &CmdDump{}) + if err != nil { + panic(err) + } +} + +func dumpIndex(r *repository.Repository, wr io.Writer) error { + fmt.Fprintln(wr, "foo") + return nil +} + +func (cmd CmdDump) Usage() string { + return "[index|snapshots|trees|all]" +} + +func prettyPrintJSON(wr io.Writer, item interface{}) error { + buf, err := json.MarshalIndent(item, "", " ") + if err != nil { + return err + } + + _, err = wr.Write(append(buf, '\n')) + return err +} + +func printSnapshots(repo *repository.Repository, wr io.Writer) error { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Snapshot, done) { + snapshot, err := restic.LoadSnapshot(repo, id) + if err != nil { + fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err) + continue + } + + fmt.Fprintf(wr, "snapshot_id: %v\n", id) + + err = prettyPrintJSON(wr, snapshot) + if err != nil { + return err + } + } + + return nil +} + +func printTrees(repo *repository.Repository, wr io.Writer) error { + done := make(chan struct{}) + defer close(done) + + trees := []backend.ID{} + + for blob := range repo.Index().Each(done) { + if blob.Type != pack.Tree { + continue + } + + trees = append(trees, blob.ID) + } + + for _, id := range trees { + tree, err := restic.LoadTree(repo, id) + if err != nil { + fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err) + continue + } + + fmt.Fprintf(wr, "tree_id: %v\n", id) + + prettyPrintJSON(wr, tree) + } + + return nil +} + +func (cmd CmdDump) Execute(args []string) error { + if len(args) != 1 { + return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) + } + + repo, err := OpenRepo() + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + tpe := args[0] + + switch tpe { + case "index": + return repo.Index().Dump(os.Stdout) + case "snapshots": + return printSnapshots(repo, os.Stdout) + case "trees": + return printTrees(repo, os.Stdout) + case "all": + fmt.Printf("snapshots:\n") + err := printSnapshots(repo, os.Stdout) + if err != nil { + return err + } + + fmt.Printf("\ntrees:\n") + + err = printTrees(repo, os.Stdout) + if err != nil { + return err + } + + fmt.Printf("\nindex:\n") + + err = repo.Index().Dump(os.Stdout) + if err != nil { + return err + } + + return nil + default: + return errors.Errorf("no such type %q", tpe) + } +} From 0314d2b0c3a67341373dc000ca7c79c731d84714 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 17 May 2015 23:41:53 +0200 Subject: [PATCH 7/7] fsck: Fix check for orphaned blobs --- cmd/restic/cmd_fsck.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index f79fb8e56..1f84a3b6e 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -57,7 +57,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint return 0, err } } else { - // test if data blob is there + // test if pack for data blob is there ok, err := repo.Backend().Test(backend.Data, packID.String()) if err != nil { return 0, err @@ -70,6 +70,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint // if orphan check is active, record storage id if opts.o_data != nil { + debug.Log("restic.fsck", " recording blob %v as used\n", id) opts.o_data.Insert(id) } } @@ -235,14 +236,16 @@ func (cmd CmdFsck) Execute(args []string) error { debug.Log("restic.fsck", "starting orphaned check\n") cnt := make(map[pack.BlobType]*backend.IDSet) - cnt[pack.Data] = backend.NewIDSet() - cnt[pack.Tree] = backend.NewIDSet() + cnt[pack.Data] = cmd.o_data + cnt[pack.Tree] = cmd.o_trees for blob := range s.Index().Each(done) { - fmt.Println(blob.ID) + debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID) err = cnt[blob.Type].Find(blob.ID) if err != nil { + debug.Log("restic.fsck", " blob %v is orphaned\n", blob.ID) + if !cmd.RemoveOrphaned { fmt.Printf("orphaned %v blob %v\n", blob.Type, blob.ID) continue