diff --git a/src/restic/index/index.go b/src/restic/index/index.go index 232320615..bbd5c981b 100644 --- a/src/restic/index/index.go +++ b/src/restic/index/index.go @@ -2,6 +2,7 @@ package index import ( + "errors" "fmt" "os" "restic/backend" @@ -17,7 +18,7 @@ type Pack struct { Entries []pack.Blob } -// Blob contains informaiton about a blob. +// Blob contains information about a blob. type Blob struct { Size int64 Packs backend.IDSet @@ -25,14 +26,16 @@ type Blob struct { // Index contains information about blobs and packs stored in a repo. type Index struct { - Packs map[backend.ID]Pack - Blobs map[pack.Handle]Blob + Packs map[backend.ID]Pack + Blobs map[pack.Handle]Blob + IndexIDs backend.IDSet } func newIndex() *Index { return &Index{ - Packs: make(map[backend.ID]Pack), - Blobs: make(map[pack.Handle]Blob), + Packs: make(map[backend.ID]Pack), + Blobs: make(map[pack.Handle]Blob), + IndexIDs: backend.NewIDSet(), } } @@ -144,11 +147,16 @@ func Load(repo *repository.Repository) (*Index, error) { } results[id] = res + index.IndexIDs.Insert(id) } for superID, list := range supersedes { for indexID := range list { + if _, ok := results[indexID]; !ok { + continue + } debug.Log("index.Load", " removing index %v, superseded by %v", indexID.Str(), superID.Str()) + fmt.Fprintf(os.Stderr, "index %v can be removed, superseded by index %v\n", indexID.Str(), superID.Str()) delete(results, indexID) } } @@ -216,3 +224,73 @@ func (idx *Index) PacksForBlobs(blobs pack.BlobSet) (packs backend.IDSet) { return packs } + +// Location describes the location of a blob in a pack. +type Location struct { + PackID backend.ID + pack.Blob +} + +// ErrBlobNotFound is return by FindBlob when the blob could not be found in +// the index. +var ErrBlobNotFound = errors.New("blob not found in index") + +// FindBlob returns a list of packs and positions the blob can be found in. +func (idx *Index) FindBlob(h pack.Handle) ([]Location, error) { + blob, ok := idx.Blobs[h] + if !ok { + return nil, ErrBlobNotFound + } + + result := make([]Location, 0, len(blob.Packs)) + for packID := range blob.Packs { + pack, ok := idx.Packs[packID] + if !ok { + return nil, fmt.Errorf("pack %v not found in index", packID.Str()) + } + + for _, entry := range pack.Entries { + if entry.Type != h.Type { + continue + } + + if !entry.ID.Equal(h.ID) { + continue + } + + loc := Location{PackID: packID, Blob: entry} + result = append(result, loc) + } + } + + return result, nil +} + +// Save writes a new index containing the given packs. +func Save(repo *repository.Repository, packs map[backend.ID][]pack.Blob, supersedes backend.IDs) (backend.ID, error) { + idx := &indexJSON{ + Supersedes: supersedes, + Packs: make([]*packJSON, 0, len(packs)), + } + + for packID, blobs := range packs { + b := make([]blobJSON, 0, len(blobs)) + for _, blob := range blobs { + b = append(b, blobJSON{ + ID: blob.ID, + Type: blob.Type, + Offset: blob.Offset, + Length: blob.Length, + }) + } + + p := &packJSON{ + ID: packID, + Blobs: b, + } + + idx.Packs = append(idx.Packs, p) + } + + return repo.SaveJSONUnpacked(backend.Index, idx) +} diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go index a5e56797e..2ae33a706 100644 --- a/src/restic/index/index_test.go +++ b/src/restic/index/index_test.go @@ -1,9 +1,11 @@ package index import ( + "math/rand" "restic" "restic/backend" "restic/backend/local" + "restic/pack" "restic/repository" "testing" "time" @@ -176,3 +178,62 @@ func TestIndexDuplicateBlobs(t *testing.T) { } t.Logf("%d packs with duplicate blobs", len(packs)) } + +func loadIndex(t testing.TB, repo *repository.Repository) *Index { + idx, err := Load(repo) + if err != nil { + t.Fatalf("Load() returned error %v", err) + } + + return idx +} + +func TestIndexSave(t *testing.T) { + repo, cleanup := createFilledRepo(t, 3, 0) + defer cleanup() + + idx := loadIndex(t, repo) + + packs := make(map[backend.ID][]pack.Blob) + for id := range idx.Packs { + if rand.Float32() < 0.5 { + packs[id] = idx.Packs[id].Entries + } + } + + t.Logf("save %d/%d packs in a new index\n", len(packs), len(idx.Packs)) + + id, err := Save(repo, packs, idx.IndexIDs.List()) + if err != nil { + t.Fatalf("unable to save new index: %v", err) + } + + t.Logf("new index saved as %v", id.Str()) + + for id := range idx.IndexIDs { + t.Logf("remove index %v", id.Str()) + err = repo.Backend().Remove(backend.Index, id.String()) + if err != nil { + t.Errorf("error removing index %v: %v", id, err) + } + } + + idx2 := loadIndex(t, repo) + t.Logf("load new index with %d packs", len(idx2.Packs)) + + if len(idx2.Packs) != len(packs) { + t.Errorf("wrong number of packs in new index, want %d, got %d", len(packs), len(idx2.Packs)) + } + + for id := range packs { + if _, ok := idx2.Packs[id]; !ok { + t.Errorf("pack %v is not contained in new index", id.Str()) + } + } + + for id := range idx2.Packs { + if _, ok := packs[id]; !ok { + t.Errorf("pack %v is not contained in new index", id.Str()) + } + } +}