restic/internal/repository/master_index_test.go

389 lines
9.2 KiB
Go
Raw Normal View History

package repository_test
import (
"context"
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
"fmt"
"math/rand"
"testing"
"time"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestMasterIndex(t *testing.T) {
idInIdx1 := restic.NewRandomID()
idInIdx2 := restic.NewRandomID()
idInIdx12 := restic.NewRandomID()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx1,
Length: uint(restic.CiphertextLength(10)),
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx2,
Length: uint(restic.CiphertextLength(100)),
Offset: 10,
},
}
blob12a := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.TreeBlob,
ID: idInIdx12,
Length: uint(restic.CiphertextLength(123)),
Offset: 110,
},
}
blob12b := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.TreeBlob,
ID: idInIdx12,
Length: uint(restic.CiphertextLength(123)),
Offset: 50,
},
}
idx1 := repository.NewIndex()
idx1.Store(blob1)
idx1.Store(blob12a)
idx2 := repository.NewIndex()
idx2.Store(blob2)
idx2.Store(blob12b)
mIdx := repository.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
// test idInIdx1
found := mIdx.Has(idInIdx1, restic.DataBlob)
rtest.Equals(t, true, found)
blobs := mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
size, found := mIdx.LookupSize(idInIdx1, restic.DataBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(10), size)
// test idInIdx2
found = mIdx.Has(idInIdx2, restic.DataBlob)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
size, found = mIdx.LookupSize(idInIdx2, restic.DataBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(100), size)
// test idInIdx12
found = mIdx.Has(idInIdx12, restic.TreeBlob)
rtest.Equals(t, true, found)
blobs = mIdx.Lookup(idInIdx12, restic.TreeBlob)
rtest.Equals(t, 2, len(blobs))
// test Lookup result for blob12a
found = false
if blobs[0] == blob12a || blobs[1] == blob12a {
found = true
}
rtest.Assert(t, found, "blob12a not found in result")
// test Lookup result for blob12b
found = false
if blobs[0] == blob12b || blobs[1] == blob12b {
found = true
}
rtest.Assert(t, found, "blob12a not found in result")
size, found = mIdx.LookupSize(idInIdx12, restic.TreeBlob)
rtest.Equals(t, true, found)
rtest.Equals(t, uint(123), size)
// test not in index
found = mIdx.Has(restic.NewRandomID(), restic.TreeBlob)
rtest.Assert(t, !found, "Expected no blobs when fetching with a random id")
blobs = mIdx.Lookup(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
2020-03-06 22:32:12 +00:00
_, found = mIdx.LookupSize(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, !found, "Expected no blobs when fetching with a random id")
// Test Count
num := mIdx.Count(restic.DataBlob)
rtest.Equals(t, uint(2), num)
num = mIdx.Count(restic.TreeBlob)
rtest.Equals(t, uint(2), num)
}
func TestMasterMergeFinalIndexes(t *testing.T) {
idInIdx1 := restic.NewRandomID()
idInIdx2 := restic.NewRandomID()
blob1 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx1,
Length: 10,
Offset: 0,
},
}
blob2 := restic.PackedBlob{
PackID: restic.NewRandomID(),
Blob: restic.Blob{
Type: restic.DataBlob,
ID: idInIdx2,
Length: 100,
Offset: 10,
},
}
idx1 := repository.NewIndex()
idx1.Store(blob1)
idx2 := repository.NewIndex()
idx2.Store(blob2)
mIdx := repository.NewMasterIndex()
mIdx.Insert(idx1)
mIdx.Insert(idx2)
finalIndexes := mIdx.FinalizeNotFinalIndexes()
rtest.Equals(t, []*repository.Index{idx1, idx2}, finalIndexes)
mIdx.MergeFinalIndexes()
allIndexes := mIdx.All()
rtest.Equals(t, 1, len(allIndexes))
blobCount := 0
2020-03-06 22:32:12 +00:00
for range mIdx.Each(context.TODO()) {
blobCount++
}
rtest.Equals(t, 2, blobCount)
blobs := mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
blobs = mIdx.Lookup(restic.NewRandomID(), restic.DataBlob)
rtest.Assert(t, blobs == nil, "Expected no blobs when fetching with a random id")
// merge another index containing identical blobs
idx3 := repository.NewIndex()
idx3.Store(blob1)
idx3.Store(blob2)
mIdx.Insert(idx3)
finalIndexes = mIdx.FinalizeNotFinalIndexes()
rtest.Equals(t, []*repository.Index{idx3}, finalIndexes)
mIdx.MergeFinalIndexes()
allIndexes = mIdx.All()
rtest.Equals(t, 1, len(allIndexes))
// Index should have same entries as before!
blobs = mIdx.Lookup(idInIdx1, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob1}, blobs)
blobs = mIdx.Lookup(idInIdx2, restic.DataBlob)
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
blobCount = 0
2020-03-06 22:32:12 +00:00
for range mIdx.Each(context.TODO()) {
blobCount++
}
rtest.Equals(t, 2, blobCount)
}
func createRandomMasterIndex(rng *rand.Rand, num, size int) (*repository.MasterIndex, restic.ID) {
mIdx := repository.NewMasterIndex()
for i := 0; i < num-1; i++ {
idx, _ := createRandomIndex(rng, size)
mIdx.Insert(idx)
}
idx1, lookupID := createRandomIndex(rng, size)
mIdx.Insert(idx1)
mIdx.FinalizeNotFinalIndexes()
mIdx.MergeFinalIndexes()
return mIdx, lookupID
}
2020-08-05 04:28:24 +00:00
func BenchmarkMasterIndexAlloc(b *testing.B) {
rng := rand.New(rand.NewSource(0))
b.ReportAllocs()
for i := 0; i < b.N; i++ {
createRandomMasterIndex(rng, 10000, 5)
}
}
func BenchmarkMasterIndexLookupSingleIndex(b *testing.B) {
mIdx, lookupID := createRandomMasterIndex(rand.New(rand.NewSource(0)), 1, 200000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupMultipleIndex(b *testing.B) {
mIdx, lookupID := createRandomMasterIndex(rand.New(rand.NewSource(0)), 100, 10000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupSingleIndexUnknown(b *testing.B) {
lookupID := restic.NewRandomID()
mIdx, _ := createRandomMasterIndex(rand.New(rand.NewSource(0)), 1, 200000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
func BenchmarkMasterIndexLookupMultipleIndexUnknown(b *testing.B) {
lookupID := restic.NewRandomID()
mIdx, _ := createRandomMasterIndex(rand.New(rand.NewSource(0)), 100, 10000)
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.Lookup(lookupID, restic.DataBlob)
}
}
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
func BenchmarkMasterIndexLookupParallel(b *testing.B) {
mIdx := repository.NewMasterIndex()
for _, numindices := range []int{25, 50, 100} {
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
var lookupID restic.ID
b.StopTimer()
rng := rand.New(rand.NewSource(0))
mIdx, lookupID = createRandomMasterIndex(rng, numindices, 10000)
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
b.StartTimer()
name := fmt.Sprintf("known,indices=%d", numindices)
b.Run(name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
mIdx.Lookup(lookupID, restic.DataBlob)
}
})
})
lookupID = restic.NewRandomID()
name = fmt.Sprintf("unknown,indices=%d", numindices)
b.Run(name, func(b *testing.B) {
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
mIdx.Lookup(lookupID, restic.DataBlob)
}
})
})
}
}
func BenchmarkMasterIndexLookupBlobSize(b *testing.B) {
rng := rand.New(rand.NewSource(0))
mIdx, lookupID := createRandomMasterIndex(rand.New(rng), 5, 200000)
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
b.ResetTimer()
for i := 0; i < b.N; i++ {
mIdx.LookupSize(lookupID, restic.DataBlob)
Parallel index benchmarks + benchmark optimizations createRandomIndex was using the global RNG, which locks on every call It was also using twice as many random numbers as necessary and doing a float division in every iteration of the inner loop. BenchmarkDecodeIndex was using too short an input, especially for a parallel version. (It may now be using one that is a bit large.) Results on linux/amd64, -benchtime=3s -count=20: name old time/op new time/op delta PackerManager-8 178ms ± 0% 178ms ± 0% ~ (p=0.165 n=20+20) DecodeIndex-8 13.6µs ± 2% 4539886.8µs ± 0% +33293901.38% (p=0.000 n=20+18) IndexHasUnknown-8 44.4ns ± 7% 44.4ns ± 5% ~ (p=0.873 n=20+19) IndexHasKnown-8 49.2ns ± 3% 48.3ns ± 0% -1.86% (p=0.000 n=20+16) IndexAlloc-8 802ms ± 1% 758ms ± 1% -5.51% (p=0.000 n=20+19) MasterIndexLookupSingleIndex-8 124ns ± 1% 122ns ± 0% -1.41% (p=0.000 n=20+14) MasterIndexLookupMultipleIndex-8 373ns ± 2% 369ns ± 2% -1.13% (p=0.001 n=20+20) MasterIndexLookupSingleIndexUnknown-8 67.8ns ± 3% 68.4ns ± 5% ~ (p=0.753 n=20+20) MasterIndexLookupMultipleIndexUnknown-8 316ns ± 3% 315ns ± 3% ~ (p=0.846 n=20+20) SaveAndEncrypt-8 30.5ms ± 1% 30.2ms ± 1% -1.09% (p=0.000 n=19+19) LoadTree-8 527µs ± 1% 540µs ± 1% +2.37% (p=0.000 n=19+20) LoadBlob-8 5.65ms ± 0% 5.64ms ± 0% -0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 7.07ms ± 2% 5.93ms ± 0% -16.15% (p=0.000 n=19+20) LoadIndex-8 32.1ms ± 2% 25.1ms ± 0% -21.64% (p=0.000 n=20+18) name old speed new speed delta PackerManager-8 296MB/s ± 0% 296MB/s ± 0% ~ (p=0.159 n=20+20) SaveAndEncrypt-8 138MB/s ± 1% 139MB/s ± 1% +1.10% (p=0.000 n=19+19) LoadBlob-8 177MB/s ± 0% 177MB/s ± 0% +0.21% (p=0.000 n=19+18) LoadAndDecrypt-8 141MB/s ± 2% 169MB/s ± 0% +19.24% (p=0.000 n=19+20) name old alloc/op new alloc/op delta PackerManager-8 91.8kB ± 0% 91.8kB ± 0% ~ (p=0.826 n=19+12) IndexAlloc-8 786MB ± 0% 786MB ± 0% +0.01% (p=0.000 n=20+20) SaveAndEncrypt-8 21.0MB ± 0% 21.0MB ± 0% -0.00% (p=0.012 n=20+19) name old allocs/op new allocs/op delta PackerManager-8 1.41k ± 0% 1.41k ± 0% ~ (all equal) IndexAlloc-8 977k ± 0% 977k ± 0% +0.01% (p=0.022 n=20+20) SaveAndEncrypt-8 73.0 ± 0% 73.0 ± 0% ~ (all equal)
2020-07-05 06:37:34 +00:00
}
}
var (
snapshotTime = time.Unix(1470492820, 207401672)
depth = 3
)
func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Repository, func()) {
repo, cleanup := repository.TestRepository(t)
for i := 0; i < 3; i++ {
restic.TestCreateSnapshot(t, repo, snapshotTime.Add(time.Duration(i)*time.Second), depth, dup)
}
return repo, cleanup
}
func TestIndexSave(t *testing.T) {
repo, cleanup := createFilledRepo(t, 3, 0)
defer cleanup()
repo.LoadIndex(context.TODO())
2020-10-18 07:24:34 +00:00
obsoletes, err := repo.Index().(*repository.MasterIndex).Save(context.TODO(), repo, nil, nil, nil)
if err != nil {
t.Fatalf("unable to save new index: %v", err)
}
for id := range obsoletes {
t.Logf("remove index %v", id.Str())
h := restic.Handle{Type: restic.IndexFile, Name: id.String()}
err = repo.Backend().Remove(context.TODO(), h)
if err != nil {
t.Errorf("error removing index %v: %v", id, err)
}
}
checker := checker.New(repo, false)
hints, errs := checker.LoadIndex(context.TODO())
for _, h := range hints {
t.Logf("hint: %v\n", h)
}
for _, err := range errs {
t.Errorf("checker found error: %v", err)
}
ctx, cancel := context.WithCancel(context.TODO())
defer cancel()
errCh := make(chan error)
go checker.Structure(ctx, errCh)
i := 0
for err := range errCh {
t.Errorf("checker returned error: %v", err)
i++
if i == 10 {
t.Errorf("more than 10 errors returned, skipping the rest")
cancel()
break
}
}
}