mirror of
https://github.com/restic/restic.git
synced 2024-12-23 08:16:36 +00:00
index: Limit index files to 3000 pack files
This commit is contained in:
parent
ca3c566a99
commit
f63c7048c7
3 changed files with 45 additions and 77 deletions
|
@ -73,12 +73,12 @@ func rebuildIndex(ctx context.Context, repo restic.Repository, ignorePacks resti
|
|||
return err
|
||||
}
|
||||
|
||||
id, err := idx.Save(ctx, repo, supersedes)
|
||||
ids, err := idx.Save(ctx, repo, supersedes)
|
||||
if err != nil {
|
||||
return errors.Fatalf("unable to save index, last error was: %v", err)
|
||||
}
|
||||
|
||||
Verbosef("saved new index as %v\n", id.Str())
|
||||
Verbosef("saved new indexes as %v\n", ids)
|
||||
|
||||
Verbosef("remove %d old index files\n", len(supersedes))
|
||||
|
||||
|
|
|
@ -85,8 +85,8 @@ type blobJSON struct {
|
|||
}
|
||||
|
||||
type indexJSON struct {
|
||||
Supersedes restic.IDs `json:"supersedes,omitempty"`
|
||||
Packs []*packJSON `json:"packs"`
|
||||
Supersedes restic.IDs `json:"supersedes,omitempty"`
|
||||
Packs []packJSON `json:"packs"`
|
||||
}
|
||||
|
||||
func loadIndexJSON(ctx context.Context, repo restic.Repository, id restic.ID) (*indexJSON, error) {
|
||||
|
@ -257,26 +257,24 @@ func (idx *Index) FindBlob(h restic.BlobHandle) (result []Location, err error) {
|
|||
return result, nil
|
||||
}
|
||||
|
||||
const maxEntries = 3000
|
||||
|
||||
// Save writes the complete index to the repo.
|
||||
func (idx *Index) Save(ctx context.Context, repo restic.Repository, supersedes restic.IDs) (restic.ID, error) {
|
||||
packs := make(map[restic.ID][]restic.Blob, len(idx.Packs))
|
||||
for id, p := range idx.Packs {
|
||||
packs[id] = p.Entries
|
||||
}
|
||||
func (idx *Index) Save(ctx context.Context, repo restic.Repository, supersedes restic.IDs) (restic.IDs, error) {
|
||||
debug.Log("pack files: %d\n", len(idx.Packs))
|
||||
|
||||
return Save(ctx, repo, packs, supersedes)
|
||||
}
|
||||
var indexIDs []restic.ID
|
||||
|
||||
// Save writes a new index containing the given packs.
|
||||
func Save(ctx context.Context, repo restic.Repository, packs map[restic.ID][]restic.Blob, supersedes restic.IDs) (restic.ID, error) {
|
||||
idx := &indexJSON{
|
||||
packs := 0
|
||||
jsonIDX := &indexJSON{
|
||||
Supersedes: supersedes,
|
||||
Packs: make([]*packJSON, 0, len(packs)),
|
||||
Packs: make([]packJSON, 0, maxEntries),
|
||||
}
|
||||
|
||||
for packID, blobs := range packs {
|
||||
b := make([]blobJSON, 0, len(blobs))
|
||||
for _, blob := range blobs {
|
||||
for packID, pack := range idx.Packs {
|
||||
debug.Log("%04d add pack %v with %d entries", packs, packID, len(pack.Entries))
|
||||
b := make([]blobJSON, 0, len(pack.Entries))
|
||||
for _, blob := range pack.Entries {
|
||||
b = append(b, blobJSON{
|
||||
ID: blob.ID,
|
||||
Type: blob.Type,
|
||||
|
@ -285,13 +283,35 @@ func Save(ctx context.Context, repo restic.Repository, packs map[restic.ID][]res
|
|||
})
|
||||
}
|
||||
|
||||
p := &packJSON{
|
||||
p := packJSON{
|
||||
ID: packID,
|
||||
Blobs: b,
|
||||
}
|
||||
|
||||
idx.Packs = append(idx.Packs, p)
|
||||
jsonIDX.Packs = append(jsonIDX.Packs, p)
|
||||
|
||||
packs++
|
||||
if packs == maxEntries {
|
||||
id, err := repo.SaveJSONUnpacked(ctx, restic.IndexFile, jsonIDX)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
debug.Log("saved new index as %v", id)
|
||||
|
||||
indexIDs = append(indexIDs, id)
|
||||
packs = 0
|
||||
jsonIDX.Packs = jsonIDX.Packs[:0]
|
||||
}
|
||||
}
|
||||
|
||||
return repo.SaveJSONUnpacked(ctx, restic.IndexFile, idx)
|
||||
if packs > 0 {
|
||||
id, err := repo.SaveJSONUnpacked(ctx, restic.IndexFile, jsonIDX)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
debug.Log("saved new index as %v", id)
|
||||
indexIDs = append(indexIDs, id)
|
||||
}
|
||||
|
||||
return indexIDs, nil
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@ package index
|
|||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
@ -177,12 +176,12 @@ func BenchmarkIndexSave(b *testing.B) {
|
|||
b.ResetTimer()
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
id, err := idx.Save(context.TODO(), repo, nil)
|
||||
ids, err := idx.Save(context.TODO(), repo, nil)
|
||||
if err != nil {
|
||||
b.Fatalf("New() returned error %v", err)
|
||||
}
|
||||
|
||||
b.Logf("saved as %v", id.Str())
|
||||
b.Logf("saved as %v", ids)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -217,69 +216,18 @@ func loadIndex(t testing.TB, repo restic.Repository) *Index {
|
|||
return idx
|
||||
}
|
||||
|
||||
func TestSave(t *testing.T) {
|
||||
repo, cleanup := createFilledRepo(t, 3, 0)
|
||||
defer cleanup()
|
||||
|
||||
idx := loadIndex(t, repo)
|
||||
|
||||
packs := make(map[restic.ID][]restic.Blob)
|
||||
for id := range idx.Packs {
|
||||
if rand.Float32() < 0.5 {
|
||||
packs[id] = idx.Packs[id].Entries
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("save %d/%d packs in a new index\n", len(packs), len(idx.Packs))
|
||||
|
||||
id, err := Save(context.TODO(), repo, packs, idx.IndexIDs.List())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to save new index: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("new index saved as %v", id.Str())
|
||||
|
||||
for id := range idx.IndexIDs {
|
||||
t.Logf("remove index %v", id.Str())
|
||||
h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
|
||||
err = repo.Backend().Remove(context.TODO(), h)
|
||||
if err != nil {
|
||||
t.Errorf("error removing index %v: %v", id, err)
|
||||
}
|
||||
}
|
||||
|
||||
idx2 := loadIndex(t, repo)
|
||||
t.Logf("load new index with %d packs", len(idx2.Packs))
|
||||
|
||||
if len(idx2.Packs) != len(packs) {
|
||||
t.Errorf("wrong number of packs in new index, want %d, got %d", len(packs), len(idx2.Packs))
|
||||
}
|
||||
|
||||
for id := range packs {
|
||||
if _, ok := idx2.Packs[id]; !ok {
|
||||
t.Errorf("pack %v is not contained in new index", id.Str())
|
||||
}
|
||||
}
|
||||
|
||||
for id := range idx2.Packs {
|
||||
if _, ok := packs[id]; !ok {
|
||||
t.Errorf("pack %v is not contained in new index", id.Str())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexSave(t *testing.T) {
|
||||
repo, cleanup := createFilledRepo(t, 3, 0)
|
||||
defer cleanup()
|
||||
|
||||
idx := loadIndex(t, repo)
|
||||
|
||||
id, err := idx.Save(context.TODO(), repo, idx.IndexIDs.List())
|
||||
ids, err := idx.Save(context.TODO(), repo, idx.IndexIDs.List())
|
||||
if err != nil {
|
||||
t.Fatalf("unable to save new index: %v", err)
|
||||
}
|
||||
|
||||
t.Logf("new index saved as %v", id.Str())
|
||||
t.Logf("new index saved as %v", ids)
|
||||
|
||||
for id := range idx.IndexIDs {
|
||||
t.Logf("remove index %v", id.Str())
|
||||
|
|
Loading…
Reference in a new issue