1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2024-12-25 09:18:55 +00:00
restic/internal/repository/master_index_test.go
2020-09-05 10:06:23 +02:00

324 lines
7.7 KiB
Go

package repository_test
import (
"context"
"fmt"
"math/rand"
"testing"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestMasterIndex(t *testing.T) {
idInIdx1 := restic.NewRandomID()
idInIdx2 := restic.NewRandomID()
idInIdx12 := restic.NewRandomID()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx1,
Length: uint(restic.CiphertextLength(10)),
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx2,
Length: uint(restic.CiphertextLength(100)),
Offset: 10,
},
}
blob12a := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.TreeBlob,
ID: idInIdx12,
Length: uint(restic.CiphertextLength(123)),
Offset: 110,
},
}
blob12b := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.TreeBlob,
ID: idInIdx12,
Length: uint(restic.CiphertextLength(123)),
Offset: 50,
},
}
idx1 := repository.NewIndex()
idx1.Store(blob1)
idx1.Store(blob12a)
idx2 := repository.NewIndex()
idx2.Store(blob2)
idx2.Store(blob12b)
mIdx := repository.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
// test idInIdx1
found := mIdx.Has(idInIdx1, restic.DataBlob)
rtest.Equals(t, true, found)
blobs := mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
size, found := mIdx.LookupSize(idInIdx1, restic.DataBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(10), size)
// test idInIdx2
found = mIdx.Has(idInIdx2, restic.DataBlob)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
size, found = mIdx.LookupSize(idInIdx2, restic.DataBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(100), size)
// test idInIdx12
found = mIdx.Has(idInIdx12, restic.TreeBlob)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(idInIdx12, restic.TreeBlob)
rtest.Equals(t, 2, len(blobs))
// test Lookup result for blob12a
found = false
if blobs[0] == blob12a || blobs[1] == blob12a {
found = true
}
rtest.Assert(t, found, "blob12a not found in result")
// test Lookup result for blob12b
found = false
if blobs[0] == blob12b || blobs[1] == blob12b {
found = true
}
rtest.Assert(t, found, "blob12a not found in result")
size, found = mIdx.LookupSize(idInIdx12, restic.TreeBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(123), size)
// test not in index
found = mIdx.Has(restic.NewRandomID(), restic.TreeBlob)
rtest.Assert(t, !found, "Expected no blobs when fetching with a random id")
blobs = mIdx.Lookup(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
_, found = mIdx.LookupSize(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, !found, "Expected no blobs when fetching with a random id")
// Test Count
num := mIdx.Count(restic.DataBlob)
rtest.Equals(t, uint(2), num)
num = mIdx.Count(restic.TreeBlob)
rtest.Equals(t, uint(2), num)
}
func TestMasterMergeFinalIndexes(t *testing.T) {
idInIdx1 := restic.NewRandomID()
idInIdx2 := restic.NewRandomID()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx1,
Length: 10,
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx2,
Length: 100,
Offset: 10,
},
}
idx1 := repository.NewIndex()
idx1.Store(blob1)
idx2 := repository.NewIndex()
idx2.Store(blob2)
mIdx := repository.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
finalIndexes := mIdx.FinalizeNotFinalIndexes()
rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes)
mIdx.MergeFinalIndexes()
allIndexes := mIdx.All()
rtest.Equals(t, 1, len(allIndexes))
blobCount := 0
for range mIdx.Each(context.TODO()) {
blobCount++
}
rtest.Equals(t, 2, blobCount)
blobs := mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
blobs = mIdx.Lookup(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
// merge another index containing identical blobs
idx3 := repository.NewIndex()
idx3.Store(blob1)
idx3.Store(blob2)
mIdx.Insert(idx3)
finalIndexes = mIdx.FinalizeNotFinalIndexes()
rtest.Equals(t, []*repository.Index{idx3}, finalIndexes)
mIdx.MergeFinalIndexes()
allIndexes = mIdx.All()
rtest.Equals(t, 1, len(allIndexes))
// Index should have same entries as before!
blobs = mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
blobCount = 0
for range mIdx.Each(context.TODO()) {
blobCount++
}
rtest.Equals(t, 2, blobCount)
}
func createRandomMasterIndex(rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.ID) {
mIdx := repository.NewMasterIndex()
for i := 0; i < num-1; i++ {
idx, _ := createRandomIndex(rng, size)
mIdx.Insert(idx)
}
idx1, lookupID := createRandomIndex(rng, size)
mIdx.Insert(idx1)
mIdx.FinalizeNotFinalIndexes()
mIdx.MergeFinalIndexes()
return mIdx, lookupID
}
func BenchmarkMasterIndexAlloc(b *testing.B) {
rng := rand.New(rand.NewSource(0))
b.ReportAllocs()
for i := 0; i < b.N; i++ {
createRandomMasterIndex(rng, 10000, 5)
}
}
func BenchmarkMasterIndexLookupSingleIndex(b *testing.B) {
mIdx, lookupID := createRandomMasterIndex(rand.New(rand.NewSource(0)), 1, 200000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupMultipleIndex(b *testing.B) {
mIdx, lookupID := createRandomMasterIndex(rand.New(rand.NewSource(0)), 100, 10000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupSingleIndexUnknown(b *testing.B) {
lookupID := restic.NewRandomID()
mIdx, _ := createRandomMasterIndex(rand.New(rand.NewSource(0)), 1, 200000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupMultipleIndexUnknown(b *testing.B) {
lookupID := restic.NewRandomID()
mIdx, _ := createRandomMasterIndex(rand.New(rand.NewSource(0)), 100, 10000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupParallel(b *testing.B) {
mIdx := repository.NewMasterIndex()
for _, numindices := range []int{25, 50, 100} {
var lookupID restic.ID
b.StopTimer()
rng := rand.New(rand.NewSource(0))
mIdx, lookupID = createRandomMasterIndex(rng, numindices, 10000)
b.StartTimer()
name := fmt.Sprintf("known,indices=%d", numindices)
b.Run(name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
mIdx.Lookup(lookupID, restic.DataBlob)
}
})
})
lookupID = restic.NewRandomID()
name = fmt.Sprintf("unknown,indices=%d", numindices)
b.Run(name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
mIdx.Lookup(lookupID, restic.DataBlob)
}
})
})
}
}
func BenchmarkMasterIndexLookupBlobSize(b *testing.B) {
rng := rand.New(rand.NewSource(0))
mIdx, lookupID := createRandomMasterIndex(rand.New(rng), 5, 200000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.LookupSize(lookupID, restic.DataBlob)
}
}