diff --git a/archiver.go b/archiver.go index 286cd0d8f..00d299f4d 100644 --- a/archiver.go +++ b/archiver.go @@ -254,7 +254,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan st // check if all content is still available in the repository contentMissing := false for _, blob := range oldNode.blobs { - if ok, err := arch.repo.Test(backend.Data, blob.Storage.String()); !ok || err != nil { + if ok, err := arch.repo.Backend().Test(backend.Data, blob.Storage.String()); !ok || err != nil { debug.Log("Archiver.fileWorker", " %v not using old data, %v (%v) is missing", e.Path(), blob.ID.Str(), blob.Storage.Str()) contentMissing = true break diff --git a/backend/generic.go b/backend/generic.go index c7d569972..64e48ae68 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -56,18 +56,6 @@ func Find(be Lister, t Type, prefix string) (string, error) { return "", ErrNoIDPrefixFound } -// FindSnapshot takes a string and tries to find a snapshot whose ID matches -// the string as closely as possible. -func FindSnapshot(be Lister, s string) (string, error) { - // find snapshot id with prefix - name, err := Find(be, Snapshot, s) - if err != nil { - return "", err - } - - return name, nil -} - // PrefixLength returns the number of bytes required so that all prefixes of // all names of type t are unique. func PrefixLength(be Lister, t Type) (int, error) { diff --git a/cache.go b/cache.go index 5e419bf2f..089c3f257 100644 --- a/cache.go +++ b/cache.go @@ -115,7 +115,7 @@ func (c *Cache) Clear(repo *repository.Repository) error { for _, entry := range list { debug.Log("Cache.Clear", "found entry %v", entry) - if ok, err := repo.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil { + if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil { debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry) err = c.purge(backend.Snapshot, entry.Subtype, entry.ID) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index f1e116265..08a5ff46d 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -185,28 +185,20 @@ func (cmd CmdBackup) Execute(args []string) error { return err } - var ( - parentSnapshot string - parentSnapshotID backend.ID - ) + var parentSnapshotID backend.ID // Force using a parent if !cmd.Force && cmd.Parent != "" { - parentSnapshot, err = s.FindSnapshot(cmd.Parent) + parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Parent, err) } - parentSnapshotID, err = backend.ParseID(parentSnapshot) - if err != nil { - return fmt.Errorf("invalid parent snapshot id %v", parentSnapshot) - } - fmt.Printf("found parent snapshot %v\n", parentSnapshotID) } // Find last snapshot to set it as parent, if not already set - if !cmd.Force && parentSnapshot == "" { + if !cmd.Force && parentSnapshotID == nil { samePaths := func(expected, actual []string) bool { if len(expected) != len(actual) { return false @@ -221,24 +213,19 @@ func (cmd CmdBackup) Execute(args []string) error { var latest time.Time - for snapshotIDString := range s.List(backend.Snapshot, make(chan struct{})) { - snapshotID, err := backend.ParseID(snapshotIDString) - if err != nil { - return fmt.Errorf("Error with the listing of snapshots inputting invalid backend ids: %v", err) - } - - snapshot, err := restic.LoadSnapshot(s, snapshotID) + for id := range s.List(backend.Snapshot, make(chan struct{})) { + snapshot, err := restic.LoadSnapshot(s, id) if err != nil { return fmt.Errorf("Error listing snapshot: %v", err) } if snapshot.Time.After(latest) && samePaths(snapshot.Paths, target) { latest = snapshot.Time - parentSnapshotID = snapshotID - parentSnapshot = snapshotIDString + parentSnapshotID = id } } - if parentSnapshot != "" { + + if parentSnapshotID != nil { fmt.Printf("using parent snapshot %v\n", parentSnapshotID) } } diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index e10a9071b..65e60c51f 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -53,12 +53,7 @@ func (cmd CmdCat) Execute(args []string) error { } // find snapshot id with prefix - name, err := s.FindSnapshot(args[1]) - if err != nil { - return err - } - - id, err = backend.ParseID(name) + id, err = restic.FindSnapshot(s, args[1]) if err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go new file mode 100644 index 000000000..7e7a54293 --- /dev/null +++ b/cmd/restic/cmd_dump.go @@ -0,0 +1,149 @@ +// +build debug + +package main + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/juju/errors" + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/pack" + "github.com/restic/restic/repository" +) + +type CmdDump struct{} + +func init() { + _, err := parser.AddCommand("dump", + "dump data structures", + "The dump command dumps data structures from a repository as JSON documents", + &CmdDump{}) + if err != nil { + panic(err) + } +} + +func dumpIndex(r *repository.Repository, wr io.Writer) error { + fmt.Fprintln(wr, "foo") + return nil +} + +func (cmd CmdDump) Usage() string { + return "[index|snapshots|trees|all]" +} + +func prettyPrintJSON(wr io.Writer, item interface{}) error { + buf, err := json.MarshalIndent(item, "", " ") + if err != nil { + return err + } + + _, err = wr.Write(append(buf, '\n')) + return err +} + +func printSnapshots(repo *repository.Repository, wr io.Writer) error { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Snapshot, done) { + snapshot, err := restic.LoadSnapshot(repo, id) + if err != nil { + fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err) + continue + } + + fmt.Fprintf(wr, "snapshot_id: %v\n", id) + + err = prettyPrintJSON(wr, snapshot) + if err != nil { + return err + } + } + + return nil +} + +func printTrees(repo *repository.Repository, wr io.Writer) error { + done := make(chan struct{}) + defer close(done) + + trees := []backend.ID{} + + for blob := range repo.Index().Each(done) { + if blob.Type != pack.Tree { + continue + } + + trees = append(trees, blob.ID) + } + + for _, id := range trees { + tree, err := restic.LoadTree(repo, id) + if err != nil { + fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err) + continue + } + + fmt.Fprintf(wr, "tree_id: %v\n", id) + + prettyPrintJSON(wr, tree) + } + + return nil +} + +func (cmd CmdDump) Execute(args []string) error { + if len(args) != 1 { + return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) + } + + repo, err := OpenRepo() + if err != nil { + return err + } + + err = repo.LoadIndex() + if err != nil { + return err + } + + tpe := args[0] + + switch tpe { + case "index": + return repo.Index().Dump(os.Stdout) + case "snapshots": + return printSnapshots(repo, os.Stdout) + case "trees": + return printTrees(repo, os.Stdout) + case "all": + fmt.Printf("snapshots:\n") + err := printSnapshots(repo, os.Stdout) + if err != nil { + return err + } + + fmt.Printf("\ntrees:\n") + + err = printTrees(repo, os.Stdout) + if err != nil { + return err + } + + fmt.Printf("\nindex:\n") + + err = repo.Index().Dump(os.Stdout) + if err != nil { + return err + } + + return nil + default: + return errors.Errorf("no such type %q", tpe) + } +} diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 10c1de4bd..932d596cf 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -105,13 +105,8 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str return results, nil } -func (c CmdFind) findInSnapshot(repo *repository.Repository, name string) error { - debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", name, c.oldest, c.newest) - - id, err := backend.ParseID(name) - if err != nil { - return err - } +func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error { + debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest) sn, err := restic.LoadSnapshot(repo, id) if err != nil { @@ -169,7 +164,7 @@ func (c CmdFind) Execute(args []string) error { c.pattern = args[0] if c.Snapshot != "" { - snapshotID, err := backend.FindSnapshot(s, c.Snapshot) + snapshotID, err := restic.FindSnapshot(s, c.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", args[1], err) } diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index efd28956a..1f84a3b6e 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -57,8 +57,8 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint return 0, err } } else { - // test if data blob is there - ok, err := repo.Test(backend.Data, packID.String()) + // test if pack for data blob is there + ok, err := repo.Backend().Test(backend.Data, packID.String()) if err != nil { return 0, err } @@ -70,6 +70,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint // if orphan check is active, record storage id if opts.o_data != nil { + debug.Log("restic.fsck", " recording blob %v as used\n", id) opts.o_data.Insert(id) } } @@ -198,16 +199,11 @@ func (cmd CmdFsck) Execute(args []string) error { } if cmd.Snapshot != "" { - name, err := s.FindSnapshot(cmd.Snapshot) + id, err := restic.FindSnapshot(s, cmd.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err) } - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name) - } - err = fsckSnapshot(cmd, s, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) @@ -225,13 +221,7 @@ func (cmd CmdFsck) Execute(args []string) error { defer close(done) var firstErr error - for name := range s.List(backend.Snapshot, done) { - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name) - continue - } - + for id := range s.List(backend.Snapshot, done) { err = fsckSnapshot(cmd, s, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) @@ -246,14 +236,16 @@ func (cmd CmdFsck) Execute(args []string) error { debug.Log("restic.fsck", "starting orphaned check\n") cnt := make(map[pack.BlobType]*backend.IDSet) - cnt[pack.Data] = backend.NewIDSet() - cnt[pack.Tree] = backend.NewIDSet() + cnt[pack.Data] = cmd.o_data + cnt[pack.Tree] = cmd.o_trees for blob := range s.Index().Each(done) { - fmt.Println(blob.ID) + debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID) err = cnt[blob.Type].Find(blob.ID) if err != nil { + debug.Log("restic.fsck", " blob %v is orphaned\n", blob.ID) + if !cmd.RemoveOrphaned { fmt.Printf("orphaned %v blob %v\n", blob.Type, blob.ID) continue diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index d962fdffc..344d16750 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -34,20 +34,20 @@ func listKeys(s *repository.Repository) error { done := make(chan struct{}) defer close(done) - for name := range s.List(backend.Key, done) { - k, err := repository.LoadKey(s, name) + for id := range s.List(backend.Key, done) { + k, err := repository.LoadKey(s, id.String()) if err != nil { fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err) continue } var current string - if name == s.KeyName() { + if id.String() == s.KeyName() { current = "*" } else { current = " " } - tab.Rows = append(tab.Rows, []interface{}{current, name[:plen], + tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen], k.Username, k.Hostname, k.Created.Format(TimeFormat)}) } @@ -79,7 +79,7 @@ func deleteKey(repo *repository.Repository, name string) error { return errors.New("refusing to remove key currently used to access repository") } - err := repo.Remove(backend.Key, name) + err := repo.Backend().Remove(backend.Key, name) if err != nil { return err } @@ -103,7 +103,7 @@ func changePassword(s *repository.Repository) error { } // remove old key - err = s.Remove(backend.Key, s.KeyName()) + err = s.Backend().Remove(backend.Key, s.KeyName()) if err != nil { return err } @@ -133,7 +133,7 @@ func (cmd CmdKey) Execute(args []string) error { case "add": return addKey(s) case "rm": - id, err := backend.Find(s, backend.Key, args[1]) + id, err := backend.Find(s.Backend(), backend.Key, args[1]) if err != nil { return err } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index ee07bcf0d..1f75b515c 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -77,12 +77,7 @@ func (cmd CmdLs) Execute(args []string) error { return err } - name, err := backend.FindSnapshot(s, args[0]) - if err != nil { - return err - } - - id, err := backend.ParseID(name) + id, err := restic.FindSnapshot(s, args[0]) if err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 5d6589a82..3d14b17fb 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -6,7 +6,6 @@ import ( "path/filepath" "github.com/restic/restic" - "github.com/restic/restic/backend" ) type CmdRestore struct{} @@ -40,12 +39,7 @@ func (cmd CmdRestore) Execute(args []string) error { return err } - name, err := backend.FindSnapshot(s, args[0]) - if err != nil { - errx(1, "invalid id %q: %v", args[0], err) - } - - id, err := backend.ParseID(name) + id, err := restic.FindSnapshot(s, args[0]) if err != nil { errx(1, "invalid id %q: %v", args[0], err) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index f5ebcd3fd..3703884b2 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -105,13 +105,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { defer close(done) list := []*restic.Snapshot{} - for name := range s.List(backend.Snapshot, done) { - id, err := backend.ParseID(name) - if err != nil { - fmt.Fprintf(os.Stderr, "error parsing id: %v", name) - continue - } - + for id := range s.List(backend.Snapshot, done) { sn, err := restic.LoadSnapshot(s, id) if err != nil { fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) diff --git a/repository/index.go b/repository/index.go index be7904c66..29a725872 100644 --- a/repository/index.go +++ b/repository/index.go @@ -173,28 +173,24 @@ type blobJSON struct { Length uint `json:"length"` } -// Encode writes the JSON serialization of the index to the writer w. This -// serialization only contains new blobs added via idx.Store(), not old ones -// introduced via DecodeIndex(). -func (idx *Index) Encode(w io.Writer) error { - debug.Log("Index.Encode", "encoding index") - idx.m.Lock() - defer idx.m.Unlock() - +// generatePackList returns a list of packs containing only the index entries +// that selsectFn returned true for. If selectFn is nil, the list contains all +// blobs in the index. +func (idx *Index) generatePackList(selectFn func(indexEntry) bool) ([]*packJSON, error) { list := []*packJSON{} packs := make(map[string]*packJSON) for id, blob := range idx.pack { - if blob.old { + if selectFn != nil && !selectFn(blob) { continue } - debug.Log("Index.Encode", "handle blob %q", id[:8]) + debug.Log("Index.generatePackList", "handle blob %q", id[:8]) if blob.packID == nil { - debug.Log("Index.Encode", "blob %q has no packID! (type %v, offset %v, length %v)", + debug.Log("Index.generatePackList", "blob %q has no packID! (type %v, offset %v, length %v)", id[:8], blob.tpe, blob.offset, blob.length) - return fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id) + return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id) } // see if pack is already in map @@ -217,12 +213,63 @@ func (idx *Index) Encode(w io.Writer) error { }) } + debug.Log("Index.generatePackList", "done") + + return list, nil +} + +// encode writes the JSON serialization of the index filtered by selectFn to enc. +func (idx *Index) encode(w io.Writer, selectFn func(indexEntry) bool) error { + list, err := idx.generatePackList(func(entry indexEntry) bool { + return !entry.old + }) + if err != nil { + return err + } + debug.Log("Index.Encode", "done") enc := json.NewEncoder(w) return enc.Encode(list) } +// Encode writes the JSON serialization of the index to the writer w. This +// serialization only contains new blobs added via idx.Store(), not old ones +// introduced via DecodeIndex(). +func (idx *Index) Encode(w io.Writer) error { + debug.Log("Index.Encode", "encoding index") + idx.m.Lock() + defer idx.m.Unlock() + + return idx.encode(w, func(e indexEntry) bool { return !e.old }) +} + +// Dump writes the pretty-printed JSON representation of the index to w. +func (idx *Index) Dump(w io.Writer) error { + debug.Log("Index.Dump", "dumping index") + idx.m.Lock() + defer idx.m.Unlock() + + list, err := idx.generatePackList(nil) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(list, "", " ") + if err != nil { + return err + } + + _, err = w.Write(append(buf, '\n')) + if err != nil { + return err + } + + debug.Log("Index.Dump", "done") + + return nil +} + // DecodeIndex loads and unserializes an index from rd. func DecodeIndex(rd io.Reader) (*Index, error) { debug.Log("Index.DecodeIndex", "Start decoding index") diff --git a/repository/key.go b/repository/key.go index 8a2ec4530..ce8a661d6 100644 --- a/repository/key.go +++ b/repository/key.go @@ -98,7 +98,7 @@ func SearchKey(s *Repository, password string) (*Key, error) { // try all keys in repo done := make(chan struct{}) defer close(done) - for name := range s.List(backend.Key, done) { + for name := range s.Backend().List(backend.Key, done) { key, err := OpenKey(s, name, password) if err != nil { continue diff --git a/repository/repository.go b/repository/repository.go index 16c3d8a14..aad5f3c9b 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -52,12 +52,6 @@ func (s *Repository) Find(t backend.Type, prefix string) (string, error) { return backend.Find(s.be, t, prefix) } -// FindSnapshot takes a string and tries to find a snapshot whose ID matches -// the string as closely as possible. -func (s *Repository) FindSnapshot(name string) (string, error) { - return backend.FindSnapshot(s.be, name) -} - // PrefixLength returns the number of bytes required so that all prefixes of // all IDs of type t are unique. func (s *Repository) PrefixLength(t backend.Type) (int, error) { @@ -586,7 +580,7 @@ func (s *Repository) SearchKey(password string) error { // Init creates a new master key with the supplied password and initializes the // repository config. func (s *Repository) Init(password string) error { - has, err := s.Test(backend.Config, "") + has, err := s.be.Test(backend.Config, "") if err != nil { return err } @@ -637,26 +631,49 @@ func (s *Repository) Count(t backend.Type) (n uint) { return } -// Proxy methods to backend +func (s *Repository) list(t backend.Type, done <-chan struct{}, out chan<- backend.ID) { + defer close(out) + in := s.be.List(t, done) -func (s *Repository) Get(t backend.Type, name string) (io.ReadCloser, error) { - return s.be.Get(t, name) + var ( + // disable sending on the outCh until we received a job + outCh chan<- backend.ID + // enable receiving from in + inCh = in + id backend.ID + err error + ) + + for { + select { + case <-done: + return + case strID, ok := <-inCh: + if !ok { + // input channel closed, we're done + return + } + id, err = backend.ParseID(strID) + if err != nil { + // ignore invalid IDs + continue + } + + inCh = nil + outCh = out + case outCh <- id: + outCh = nil + inCh = in + } + } } -func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan string { - return s.be.List(t, done) -} +func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.ID { + outCh := make(chan backend.ID) -func (s *Repository) Test(t backend.Type, name string) (bool, error) { - return s.be.Test(t, name) -} + go s.list(t, done, outCh) -func (s *Repository) Remove(t backend.Type, name string) error { - return s.be.Remove(t, name) -} - -func (s *Repository) Close() error { - return s.be.Close() + return outCh } func (s *Repository) Delete() error { @@ -667,6 +684,6 @@ func (s *Repository) Delete() error { return errors.New("Delete() called for backend that does not implement this method") } -func (s *Repository) Location() string { - return s.be.Location() +func (s *Repository) Close() error { + return s.be.Close() } diff --git a/snapshot.go b/snapshot.go index c5abca5e1..e308f0d04 100644 --- a/snapshot.go +++ b/snapshot.go @@ -89,3 +89,15 @@ func (sn *Snapshot) fillUserInfo() error { return nil } + +// FindSnapshot takes a string and tries to find a snapshot whose ID matches +// the string as closely as possible. +func FindSnapshot(repo *repository.Repository, s string) (backend.ID, error) { + // find snapshot id with prefix + name, err := backend.Find(repo.Backend(), backend.Snapshot, s) + if err != nil { + return nil, err + } + + return backend.ParseID(name) +}