restic/internal/archiver/archiver_test.go

315 lines
7.7 KiB
Go
Raw Normal View History

2016-08-31 20:39:36 +00:00
package archiver_test
2014-11-16 20:41:05 +00:00
import (
"bytes"
2017-06-05 21:56:59 +00:00
"context"
2014-11-16 20:41:05 +00:00
"io"
"testing"
"time"
2014-11-16 20:41:05 +00:00
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/archiver"
"github.com/restic/restic/internal/checker"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/repository"
2017-07-24 15:42:25 +00:00
"github.com/restic/restic/internal/restic"
2017-07-23 12:21:03 +00:00
. "github.com/restic/restic/internal/test"
"github.com/restic/restic/internal/errors"
"github.com/restic/chunker"
2014-11-16 20:41:05 +00:00
)
var testPol = chunker.Pol(0x3DA3358B4DC173)
2015-02-17 19:02:43 +00:00
type Rdr interface {
io.ReadSeeker
io.ReaderAt
2015-02-17 19:02:43 +00:00
}
func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) {
rd.Seek(0, 0)
ch := chunker.New(rd, testPol)
for {
chunk, err := ch.Next(buf)
if errors.Cause(err) == io.EOF {
break
}
2015-04-09 19:15:48 +00:00
OK(b, err)
2015-02-17 19:02:43 +00:00
// reduce length of buf
Assert(b, uint(len(chunk.Data)) == chunk.Length,
"invalid length: got %d, expected %d", len(chunk.Data), chunk.Length)
_, err = key.Encrypt(buf2, chunk.Data)
2015-04-09 19:15:48 +00:00
OK(b, err)
}
}
2014-11-16 20:41:05 +00:00
func BenchmarkChunkEncrypt(b *testing.B) {
2016-09-04 11:24:51 +00:00
repo, cleanup := repository.TestRepository(b)
2016-09-04 10:52:43 +00:00
defer cleanup()
data := Random(23, 10<<20) // 10MiB
rd := bytes.NewReader(data)
2014-11-16 20:41:05 +00:00
2015-05-04 22:14:07 +00:00
buf := make([]byte, chunker.MaxSize)
buf2 := make([]byte, chunker.MaxSize)
2015-02-17 19:02:43 +00:00
2014-11-16 20:41:05 +00:00
b.ResetTimer()
b.SetBytes(int64(len(data)))
for i := 0; i < b.N; i++ {
benchmarkChunkEncrypt(b, buf, buf2, rd, repo.Key())
}
2015-02-17 19:02:43 +00:00
}
func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) {
ch := chunker.New(rd, testPol)
2014-11-16 20:41:05 +00:00
for {
chunk, err := ch.Next(buf)
if errors.Cause(err) == io.EOF {
break
}
2014-11-16 20:41:05 +00:00
// reduce length of chunkBuf
key.Encrypt(chunk.Data, chunk.Data)
}
}
func BenchmarkChunkEncryptParallel(b *testing.B) {
2016-09-04 11:24:51 +00:00
repo, cleanup := repository.TestRepository(b)
2016-09-04 10:52:43 +00:00
defer cleanup()
data := Random(23, 10<<20) // 10MiB
2015-05-04 22:14:07 +00:00
buf := make([]byte, chunker.MaxSize)
2015-02-17 19:02:43 +00:00
b.ResetTimer()
b.SetBytes(int64(len(data)))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
rd := bytes.NewReader(data)
benchmarkChunkEncryptP(pb, buf, rd, repo.Key())
2014-11-16 20:41:05 +00:00
}
})
2014-11-16 20:41:05 +00:00
}
func archiveDirectory(b testing.TB) {
2016-09-04 11:24:51 +00:00
repo, cleanup := repository.TestRepository(b)
2016-09-04 10:52:43 +00:00
defer cleanup()
2016-08-31 21:07:50 +00:00
arch := archiver.New(repo)
2017-06-05 21:56:59 +00:00
_, id, err := arch.Snapshot(context.TODO(), nil, []string{BenchArchiveDirectory}, nil, "localhost", nil)
2015-04-30 01:41:51 +00:00
OK(b, err)
b.Logf("snapshot archived as %v", id)
}
2015-02-17 21:39:44 +00:00
func TestArchiveDirectory(t *testing.T) {
2015-06-28 11:15:35 +00:00
if BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping TestArchiveDirectory")
}
archiveDirectory(t)
}
func BenchmarkArchiveDirectory(b *testing.B) {
2015-06-28 11:15:35 +00:00
if BenchArchiveDirectory == "" {
b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory")
}
2015-05-01 20:58:50 +00:00
for i := 0; i < b.N; i++ {
archiveDirectory(b)
}
2015-02-17 21:39:44 +00:00
}
func countPacks(repo restic.Repository, t restic.FileType) (n uint) {
2017-06-05 21:56:59 +00:00
for range repo.Backend().List(context.TODO(), t) {
n++
}
return n
}
func archiveWithDedup(t testing.TB) {
2016-09-04 11:24:51 +00:00
repo, cleanup := repository.TestRepository(t)
2016-09-04 10:52:43 +00:00
defer cleanup()
2015-06-28 11:15:35 +00:00
if BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping TestArchiverDedup")
2015-02-17 22:40:37 +00:00
}
var cnt struct {
before, after, after2 struct {
packs, dataBlobs, treeBlobs uint
}
}
2015-02-17 21:39:44 +00:00
// archive a few files
2016-09-04 10:52:43 +00:00
sn := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, nil)
2015-03-09 21:58:17 +00:00
t.Logf("archived snapshot %v", sn.ID().Str())
2015-02-17 21:39:44 +00:00
// get archive stats
cnt.before.packs = countPacks(repo, restic.DataFile)
2016-08-31 20:39:36 +00:00
cnt.before.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.before.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs)
2015-02-17 21:39:44 +00:00
2015-03-09 21:58:17 +00:00
// archive the same files again, without parent snapshot
2016-09-04 10:52:43 +00:00
sn2 := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, nil)
2015-03-09 21:58:17 +00:00
t.Logf("archived snapshot %v", sn2.ID().Str())
2015-02-17 21:39:44 +00:00
// get archive stats again
cnt.after.packs = countPacks(repo, restic.DataFile)
2016-08-31 20:39:36 +00:00
cnt.after.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.after.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.after.packs, cnt.after.dataBlobs, cnt.after.treeBlobs)
// if there are more data blobs, something is wrong
if cnt.after.dataBlobs > cnt.before.dataBlobs {
t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
cnt.before.dataBlobs, cnt.after.dataBlobs)
}
2015-03-09 21:58:17 +00:00
// archive the same files again, with a parent snapshot
2016-09-04 10:52:43 +00:00
sn3 := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, sn2.ID())
2015-03-09 21:58:17 +00:00
t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str())
// get archive stats again
cnt.after2.packs = countPacks(repo, restic.DataFile)
2016-08-31 20:39:36 +00:00
cnt.after2.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.after2.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.after2.packs, cnt.after2.dataBlobs, cnt.after2.treeBlobs)
// if there are more data blobs, something is wrong
if cnt.after2.dataBlobs > cnt.before.dataBlobs {
t.Fatalf("TestArchiverDedup: too many data blobs in repository: before %d, after %d",
cnt.before.dataBlobs, cnt.after2.dataBlobs)
}
2015-02-17 21:39:44 +00:00
}
2015-02-21 23:09:57 +00:00
func TestArchiveDedup(t *testing.T) {
archiveWithDedup(t)
}
// Saves several identical chunks concurrently and later checks that there are no
// unreferenced packs in the repository. See also #292 and #358.
func TestParallelSaveWithDuplication(t *testing.T) {
for seed := 0; seed < 10; seed++ {
testParallelSaveWithDuplication(t, seed)
}
}
func testParallelSaveWithDuplication(t *testing.T, seed int) {
2016-09-04 11:24:51 +00:00
repo, cleanup := repository.TestRepository(t)
2016-09-04 10:52:43 +00:00
defer cleanup()
dataSizeMb := 128
duplication := 7
2016-08-31 21:07:50 +00:00
arch := archiver.New(repo)
chunks := getRandomData(seed, dataSizeMb*1024*1024)
errChannels := [](<-chan error){}
2017-02-08 23:43:10 +00:00
// interwoven processing of subsequent chunks
maxParallel := 2*duplication - 1
barrier := make(chan struct{}, maxParallel)
for _, c := range chunks {
for dupIdx := 0; dupIdx < duplication; dupIdx++ {
errChan := make(chan error)
errChannels = append(errChannels, errChan)
go func(c chunker.Chunk, errChan chan<- error) {
barrier <- struct{}{}
2016-08-31 21:07:50 +00:00
id := restic.Hash(c.Data)
time.Sleep(time.Duration(id[0]))
2017-06-05 21:56:59 +00:00
err := arch.Save(context.TODO(), restic.DataBlob, c.Data, id)
<-barrier
errChan <- err
}(c, errChan)
}
}
for _, errChan := range errChannels {
OK(t, <-errChan)
}
OK(t, repo.Flush())
2017-06-05 21:56:59 +00:00
OK(t, repo.SaveIndex(context.TODO()))
chkr := createAndInitChecker(t, repo)
assertNoUnreferencedPacks(t, chkr)
}
func getRandomData(seed int, size int) []chunker.Chunk {
buf := Random(seed, size)
var chunks []chunker.Chunk
chunker := chunker.New(bytes.NewReader(buf), testPol)
for {
c, err := chunker.Next(nil)
if errors.Cause(err) == io.EOF {
break
}
chunks = append(chunks, c)
}
return chunks
}
2016-08-31 21:07:50 +00:00
func createAndInitChecker(t *testing.T, repo restic.Repository) *checker.Checker {
chkr := checker.New(repo)
2017-06-05 21:56:59 +00:00
hints, errs := chkr.LoadIndex(context.TODO())
if len(errs) > 0 {
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
}
if len(hints) > 0 {
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
}
return chkr
}
func assertNoUnreferencedPacks(t *testing.T, chkr *checker.Checker) {
errChan := make(chan error)
2017-06-05 21:56:59 +00:00
go chkr.Packs(context.TODO(), errChan)
for err := range errChan {
OK(t, err)
}
}
2017-03-07 10:17:15 +00:00
func TestArchiveEmptySnapshot(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
arch := archiver.New(repo)
2017-06-05 21:56:59 +00:00
sn, id, err := arch.Snapshot(context.TODO(), nil, []string{"file-does-not-exist-123123213123", "file2-does-not-exist-too-123123123"}, nil, "localhost", nil)
2017-03-07 10:17:15 +00:00
if err == nil {
t.Errorf("expected error for empty snapshot, got nil")
}
if !id.IsNull() {
t.Errorf("expected null ID for empty snapshot, got %v", id.Str())
}
if sn != nil {
t.Errorf("expected null snapshot for empty snapshot, got %v", sn)
}
}