restic/internal/index/index_test.go

package index

import (
	"context"
	"sync"
	"testing"
	"time"

	"github.com/restic/restic/internal/checker"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/test"
)

var (
	snapshotTime = time.Unix(1470492820, 207401672)
	depth        = 3
)

func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Repository, func()) {
	repo, cleanup := repository.TestRepository(t)

	for i := 0; i < 3; i++ {
		restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup)
	}

	return repo, cleanup
}

func validateIndex(t testing.TB, repo restic.Repository, idx *Index) {
	err := repo.List(context.TODO(), restic.DataFile, func(id restic.ID, size int64) error {
		p, ok := idx.Packs[id]
		if !ok {
			t.Errorf("pack %v missing from index", id.Str())
		}

		if !p.ID.Equal(id) {
			t.Errorf("pack %v has invalid ID: want %v, got %v", id.Str(), id, p.ID)
		}
		return nil
	})

	if err != nil {
		t.Fatal(err)
	}
}

func TestIndexNew(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	idx, invalid, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
	if err != nil {
		t.Fatalf("New() returned error %v", err)
	}

	if idx == nil {
		t.Fatalf("New() returned nil index")
	}

	if len(invalid) > 0 {
		t.Fatalf("New() returned invalid files: %v", invalid)
	}

	validateIndex(t, repo, idx)
}

type ErrorRepo struct {
	restic.Repository
	MaxListFiles int

	MaxPacks      int
	MaxPacksMutex sync.Mutex
}

// List returns an error after repo.MaxListFiles files.
func (repo *ErrorRepo) List(ctx context.Context, t restic.FileType, fn func(restic.ID, int64) error) error {
	if repo.MaxListFiles == 0 {
		return errors.New("test error, max is zero")
	}

	max := repo.MaxListFiles
	return repo.Repository.List(ctx, t, func(id restic.ID, size int64) error {
		if max == 0 {
			return errors.New("test error, max reached zero")
		}

		max--
		return fn(id, size)
	})
}

// ListPack returns an error after repo.MaxPacks files.
func (repo *ErrorRepo) ListPack(ctx context.Context, id restic.ID, size int64) ([]restic.Blob, int64, error) {
	repo.MaxPacksMutex.Lock()
	max := repo.MaxPacks
	if max > 0 {
		repo.MaxPacks--
	}
	repo.MaxPacksMutex.Unlock()

	if max == 0 {
		return nil, 0, errors.New("test list pack error")
	}

	return repo.Repository.ListPack(ctx, id, size)
}

func TestIndexNewListErrors(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	for _, max := range []int{0, 3, 5} {
		errRepo := &ErrorRepo{
			Repository:   repo,
			MaxListFiles: max,
		}
		idx, invalid, err := New(context.TODO(), errRepo, restic.NewIDSet(), nil)
		if err == nil {
			t.Errorf("expected error not found, got nil")
		}

		if idx != nil {
			t.Errorf("expected nil index, got %v", idx)
		}

		if len(invalid) != 0 {
			t.Errorf("expected empty invalid list, got %v", invalid)
		}
	}
}

func TestIndexNewPackErrors(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	for _, max := range []int{0, 3, 5} {
		errRepo := &ErrorRepo{
			Repository: repo,
			MaxPacks:   max,
		}
		idx, invalid, err := New(context.TODO(), errRepo, restic.NewIDSet(), nil)
		if err == nil {
			t.Errorf("expected error not found, got nil")
		}

		if idx != nil {
			t.Errorf("expected nil index, got %v", idx)
		}

		if len(invalid) != 0 {
			t.Errorf("expected empty invalid list, got %v", invalid)
		}
	}
}

func TestIndexLoad(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	loadIdx, err := Load(context.TODO(), repo, nil)
	if err != nil {
		t.Fatalf("Load() returned error %v", err)
	}

	if loadIdx == nil {
		t.Fatalf("Load() returned nil index")
	}

	validateIndex(t, repo, loadIdx)

	newIdx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
	if err != nil {
		t.Fatalf("New() returned error %v", err)
	}

	if len(loadIdx.Packs) != len(newIdx.Packs) {
		t.Errorf("number of packs does not match: want %v, got %v",
			len(loadIdx.Packs), len(newIdx.Packs))
	}

	validateIndex(t, repo, newIdx)

	for packID, packNew := range newIdx.Packs {
		packLoad, ok := loadIdx.Packs[packID]

		if !ok {
			t.Errorf("loaded index does not list pack %v", packID.Str())
			continue
		}

		if len(packNew.Entries) != len(packLoad.Entries) {
			t.Errorf("  number of entries in pack %v does not match: %d != %d\n  %v\n  %v",
				packID.Str(), len(packNew.Entries), len(packLoad.Entries),
				packNew.Entries, packLoad.Entries)
			continue
		}

		for _, entryNew := range packNew.Entries {
			found := false
			for _, entryLoad := range packLoad.Entries {
				if !entryLoad.ID.Equal(entryNew.ID) {
					continue
				}

				if entryLoad.Type != entryNew.Type {
					continue
				}

				if entryLoad.Offset != entryNew.Offset {
					continue
				}

				if entryLoad.Length != entryNew.Length {
					continue
				}

				found = true
				break
			}

			if !found {
				t.Errorf("blob not found in loaded index: %v", entryNew)
			}
		}
	}
}

func BenchmarkIndexNew(b *testing.B) {
	repo, cleanup := createFilledRepo(b, 3, 0)
	defer cleanup()

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)

		if err != nil {
			b.Fatalf("New() returned error %v", err)
		}

		if idx == nil {
			b.Fatalf("New() returned nil index")
		}
		b.Logf("idx %v packs", len(idx.Packs))
	}
}

func BenchmarkIndexSave(b *testing.B) {
	repo, cleanup := repository.TestRepository(b)
	defer cleanup()

	idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
	test.OK(b, err)

	for i := 0; i < 8000; i++ {
		entries := make([]restic.Blob, 0, 200)
		for j := 0; j < cap(entries); j++ {
			entries = append(entries, restic.Blob{
				ID:     restic.NewRandomID(),
				Length: 1000,
				Offset: 5,
				Type:   restic.DataBlob,
			})
		}

		idx.AddPack(restic.NewRandomID(), 10000, entries)
	}

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		ids, err := idx.Save(context.TODO(), repo, nil)
		if err != nil {
			b.Fatalf("New() returned error %v", err)
		}

		b.Logf("saved as %v", ids)
	}
}

func TestIndexDuplicateBlobs(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0.05)
	defer cleanup()

	idx, _, err := New(context.TODO(), repo, restic.NewIDSet(), nil)
	if err != nil {
		t.Fatal(err)
	}

	dups := idx.DuplicateBlobs()
	if len(dups) == 0 {
		t.Errorf("no duplicate blobs found")
	}
	t.Logf("%d packs, %d duplicate blobs", len(idx.Packs), len(dups))

	packs := idx.PacksForBlobs(dups)
	if len(packs) == 0 {
		t.Errorf("no packs with duplicate blobs found")
	}
	t.Logf("%d packs with duplicate blobs", len(packs))
}

func loadIndex(t testing.TB, repo restic.Repository) *Index {
	idx, err := Load(context.TODO(), repo, nil)
	if err != nil {
		t.Fatalf("Load() returned error %v", err)
	}

	return idx
}

func TestIndexSave(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	idx := loadIndex(t, repo)

	ids, err := idx.Save(context.TODO(), repo, idx.IndexIDs.List())
	if err != nil {
		t.Fatalf("unable to save new index: %v", err)
	}

	t.Logf("new index saved as %v", ids)

	for id := range idx.IndexIDs {
		t.Logf("remove index %v", id.Str())
		h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
		err = repo.Backend().Remove(context.TODO(), h)
		if err != nil {
			t.Errorf("error removing index %v: %v", id, err)
		}
	}

	idx2 := loadIndex(t, repo)
	t.Logf("load new index with %d packs", len(idx2.Packs))

	checker := checker.New(repo)
	hints, errs := checker.LoadIndex(context.TODO())
	for _, h := range hints {
		t.Logf("hint: %v\n", h)
	}

	for _, err := range errs {
		t.Errorf("checker found error: %v", err)
	}

	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()

	errCh := make(chan error)
	go checker.Structure(ctx, errCh)
	i := 0
	for err := range errCh {
		t.Errorf("checker returned error: %v", err)
		i++
		if i == 10 {
			t.Errorf("more than 10 errors returned, skipping the rest")
			cancel()
			break
		}
	}
}

// Location describes the location of a blob in a pack.
type location struct {
	PackID restic.ID
	restic.Blob
}

// FindBlob returns a list of packs and positions the blob can be found in.
func (idx *Index) findBlob(h restic.BlobHandle) (result []location) {
	for id, p := range idx.Packs {
		for _, entry := range p.Entries {
			if entry.ID.Equal(h.ID) && entry.Type == h.Type {
				result = append(result, location{
					PackID: id,
					Blob:   entry,
				})
			}
		}
	}

	return result
}

func TestIndexAddRemovePack(t *testing.T) {
	repo, cleanup := createFilledRepo(t, 3, 0)
	defer cleanup()

	idx, err := Load(context.TODO(), repo, nil)
	if err != nil {
		t.Fatalf("Load() returned error %v", err)
	}

	var packID restic.ID
	err = repo.List(context.TODO(), restic.DataFile, func(id restic.ID, size int64) error {
		packID = id
		return nil
	})
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("selected pack %v", packID.Str())

	blobs := idx.Packs[packID].Entries

	idx.RemovePack(packID)

	if _, ok := idx.Packs[packID]; ok {
		t.Errorf("removed pack %v found in index.Packs", packID.Str())
	}

	for _, blob := range blobs {
		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
		locs := idx.findBlob(h)
		if len(locs) != 0 {
			t.Errorf("removed blob %v found in index", h)
		}
	}
}

// example index serialization from doc/Design.rst
var docExample = []byte(`
{
  "supersedes": [
	"ed54ae36197f4745ebc4b54d10e0f623eaaaedd03013eb7ae90df881b7781452"
  ],
  "packs": [
	{
	  "id": "73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c",
	  "blobs": [
		{
		  "id": "3ec79977ef0cf5de7b08cd12b874cd0f62bbaf7f07f3497a5b1bbcc8cb39b1ce",
		  "type": "data",
		  "offset": 0,
		  "length": 25
		},{
		  "id": "9ccb846e60d90d4eb915848add7aa7ea1e4bbabfc60e573db9f7bfb2789afbae",
		  "type": "tree",
		  "offset": 38,
		  "length": 100
		},
		{
		  "id": "d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66",
		  "type": "data",
		  "offset": 150,
		  "length": 123
		}
	  ]
	}
  ]
}
`)

func TestIndexLoadDocReference(t *testing.T) {
	repo, cleanup := repository.TestRepository(t)
	defer cleanup()

	id, err := repo.SaveUnpacked(context.TODO(), restic.IndexFile, docExample)
	if err != nil {
		t.Fatalf("SaveUnpacked() returned error %v", err)
	}

	t.Logf("index saved as %v", id.Str())

	idx := loadIndex(t, repo)

	blobID := restic.TestParseID("d3dc577b4ffd38cc4b32122cabf8655a0223ed22edfd93b353dc0c3f2b0fdf66")
	locs := idx.findBlob(restic.BlobHandle{ID: blobID, Type: restic.DataBlob})
	if len(locs) == 0 {
		t.Error("blob not found in index")
	}

	if len(locs) != 1 {
		t.Errorf("blob found %d times, expected just one", len(locs))
	}

	l := locs[0]
	if !l.ID.Equal(blobID) {
		t.Errorf("blob IDs are not equal: %v != %v", l.ID, blobID)
	}

	if l.Type != restic.DataBlob {
		t.Errorf("want type %v, got %v", restic.DataBlob, l.Type)
	}

	if l.Offset != 150 {
		t.Errorf("wrong offset, want %d, got %v", 150, l.Offset)
	}

	if l.Length != 123 {
		t.Errorf("wrong length, want %d, got %v", 123, l.Length)
	}
}