From fccde030d5bb487d99533a76da55f4099d7952b8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 31 Oct 2015 14:47:42 +0100 Subject: [PATCH 1/2] Refactor Index.Lookup() to return struct PackedBlob --- cmd/restic/cmd_cat.go | 6 ++-- node.go | 6 ++-- repository/index.go | 47 ++++++++++++++++++-------- repository/index_test.go | 68 +++++++++++++++++++------------------- repository/master_index.go | 9 +++-- repository/repository.go | 25 +++++++------- 6 files changed, 90 insertions(+), 71 deletions(-) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 4e76f8237..ca5c24aed 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -162,16 +162,16 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - _, blobType, _, length, err := repo.Index().Lookup(id) + blob, err := repo.Index().Lookup(id) if err != nil { return err } - if blobType != pack.Data { + if blob.Type != pack.Data { return errors.New("wrong type for blob") } - buf := make([]byte, length) + buf := make([]byte, blob.Length) data, err := repo.LoadBlob(pack.Data, id, buf) if err != nil { return err diff --git a/node.go b/node.go index e79306d5d..eb1ca5f02 100644 --- a/node.go +++ b/node.go @@ -212,14 +212,14 @@ func (node Node) createFileAt(path string, repo *repository.Repository) error { var buf []byte for _, id := range node.Content { - _, _, _, length, err := repo.Index().Lookup(id) + blob, err := repo.Index().Lookup(id) if err != nil { return err } buf = buf[:cap(buf)] - if uint(len(buf)) < length { - buf = make([]byte, length) + if uint(len(buf)) < blob.Length { + buf = make([]byte, blob.Length) } buf, err := repo.LoadBlob(pack.Data, id, buf) diff --git a/repository/index.go b/repository/index.go index 25e4c5143..2ffac231c 100644 --- a/repository/index.go +++ b/repository/index.go @@ -109,24 +109,32 @@ func (idx *Index) Store(t pack.BlobType, id backend.ID, pack backend.ID, offset, idx.store(t, id, pack, offset, length) } -// Lookup returns the pack for the id. -func (idx *Index) Lookup(id backend.ID) (packID backend.ID, tpe pack.BlobType, offset, length uint, err error) { +// Lookup queries the index for the blob ID and returns a PackedBlob. +func (idx *Index) Lookup(id backend.ID) (pb PackedBlob, err error) { idx.m.Lock() defer idx.m.Unlock() if p, ok := idx.pack[id]; ok { debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d", id.Str(), p.packID.Str(), p.offset, p.length) - return p.packID, p.tpe, p.offset, p.length, nil + + pb := PackedBlob{ + Type: p.tpe, + Length: p.length, + ID: id, + Offset: p.offset, + PackID: p.packID, + } + return pb, nil } debug.Log("Index.Lookup", "id %v not found", id.Str()) - return backend.ID{}, pack.Data, 0, 0, fmt.Errorf("id %v not found in index", id) + return PackedBlob{}, fmt.Errorf("id %v not found in index", id) } // Has returns true iff the id is listed in the index. func (idx *Index) Has(id backend.ID) bool { - _, _, _, _, err := idx.Lookup(id) + _, err := idx.Lookup(id) if err == nil { return true } @@ -137,11 +145,11 @@ func (idx *Index) Has(id backend.ID) bool { // LookupSize returns the length of the cleartext content behind the // given id func (idx *Index) LookupSize(id backend.ID) (cleartextLength uint, err error) { - _, _, _, encryptedLength, err := idx.Lookup(id) + blob, err := idx.Lookup(id) if err != nil { return 0, err } - return encryptedLength - crypto.Extension, nil + return blob.PlaintextLength(), nil } // Merge loads all items from other into idx. @@ -181,10 +189,23 @@ func (idx *Index) AddToSupersedes(ids ...backend.ID) error { // PackedBlob is a blob already saved within a pack. type PackedBlob struct { - pack.Blob + Type pack.BlobType + Length uint + ID backend.ID + Offset uint PackID backend.ID } +func (pb PackedBlob) String() string { + return fmt.Sprintf(" uint(cap(plaintextBuf))+crypto.Extension { - return nil, fmt.Errorf("buf is too small, need %d more bytes", length-uint(cap(plaintextBuf))-crypto.Extension) + plaintextBufSize := uint(cap(plaintextBuf)) + if blob.PlaintextLength() > plaintextBufSize { + return nil, fmt.Errorf("buf is too small, need %d more bytes", blob.PlaintextLength()-plaintextBufSize) } - if tpe != t { - debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, tpe) - return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", tpe, t) + if blob.Type != t { + debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, blob.Type) + return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", blob.Type, t) } - debug.Log("Repo.LoadBlob", "id %v found in pack %v at offset %v (length %d)", id.Str(), packID.Str(), offset, length) + debug.Log("Repo.LoadBlob", "id %v found: %v", id.Str(), blob) // load blob from pack - rd, err := r.be.GetReader(backend.Data, packID.String(), offset, length) + rd, err := r.be.GetReader(backend.Data, blob.PackID.String(), blob.Offset, blob.Length) if err != nil { - debug.Log("Repo.LoadBlob", "error loading pack %v for %v: %v", packID.Str(), id.Str(), err) + debug.Log("Repo.LoadBlob", "error loading blob %v: %v", blob, err) return nil, err } // make buffer that is large enough for the complete blob - ciphertextBuf := make([]byte, length) + ciphertextBuf := make([]byte, blob.Length) _, err = io.ReadFull(rd, ciphertextBuf) if err != nil { return nil, err @@ -173,13 +174,13 @@ func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interf // data and afterwards call json.Unmarshal on the item. func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) error { // lookup pack - packID, _, offset, length, err := r.idx.Lookup(id) + blob, err := r.idx.Lookup(id) if err != nil { return err } // load blob from pack - rd, err := r.be.GetReader(backend.Data, packID.String(), offset, length) + rd, err := r.be.GetReader(backend.Data, blob.PackID.String(), blob.Offset, blob.Length) if err != nil { return err } From ed9470b19dc2b31e2ca3062d16b3b40173fe24d9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 31 Oct 2015 14:53:03 +0100 Subject: [PATCH 2/2] Remove tempfiles after test --- node_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/node_test.go b/node_test.go index f02f0e948..e43354bbc 100644 --- a/node_test.go +++ b/node_test.go @@ -15,7 +15,6 @@ import ( func BenchmarkNodeFillUser(t *testing.B) { tempfile, err := ioutil.TempFile("", "restic-test-temp-") - defer tempfile.Close() if err != nil { t.Fatal(err) } @@ -32,11 +31,13 @@ func BenchmarkNodeFillUser(t *testing.B) { for i := 0; i < t.N; i++ { restic.NodeFromFileInfo(path, fi) } + + OK(t, tempfile.Close()) + RemoveAll(t, tempfile.Name()) } func BenchmarkNodeFromFileInfo(t *testing.B) { tempfile, err := ioutil.TempFile("", "restic-test-temp-") - defer tempfile.Close() if err != nil { t.Fatal(err) } @@ -56,6 +57,9 @@ func BenchmarkNodeFromFileInfo(t *testing.B) { t.Fatal(err) } } + + OK(t, tempfile.Close()) + RemoveAll(t, tempfile.Name()) } func parseTime(s string) time.Time {