From 628fb0fb721d6de25345af7cb06f4a412df1e7a8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 1 Aug 2016 21:57:59 +0200 Subject: [PATCH] Fix TestCreateSnapshot, do not generate duplicate data --- src/restic/archive_reader_test.go | 6 +- src/restic/testing.go | 119 +++++++++++++++++++++++------- src/restic/testing_test.go | 7 +- 3 files changed, 100 insertions(+), 32 deletions(-) diff --git a/src/restic/archive_reader_test.go b/src/restic/archive_reader_test.go index 4397bb90e..80003c151 100644 --- a/src/restic/archive_reader_test.go +++ b/src/restic/archive_reader_test.go @@ -63,7 +63,7 @@ func TestArchiveReader(t *testing.T) { size := int64(rand.Intn(50*1024*1024) + 50*1024*1024) t.Logf("seed is 0x%016x, size is %v", seed, size) - f := fakeFile(t, seed, size) + f := fakeFile(seed, size) sn, id, err := ArchiveReader(repo, nil, f, "fakefile") if err != nil { @@ -76,7 +76,7 @@ func TestArchiveReader(t *testing.T) { t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str()) - checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size)) + checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(seed, size)) } func BenchmarkArchiveReader(t *testing.B) { @@ -86,7 +86,7 @@ func BenchmarkArchiveReader(t *testing.B) { const size = 50 * 1024 * 1024 buf := make([]byte, size) - _, err := io.ReadFull(fakeFile(t, 23, size), buf) + _, err := io.ReadFull(fakeFile(23, size), buf) if err != nil { t.Fatal(err) } diff --git a/src/restic/testing.go b/src/restic/testing.go index fa2ab649d..82dc67d74 100644 --- a/src/restic/testing.go +++ b/src/restic/testing.go @@ -1,6 +1,7 @@ package restic import ( + "encoding/json" "fmt" "io" "math/rand" @@ -85,14 +86,21 @@ func (rd *randReader) Read(p []byte) (int, error) { } // fakeFile returns a reader which yields deterministic pseudo-random data. -func fakeFile(t testing.TB, seed, size int64) io.Reader { +func fakeFile(seed, size int64) io.Reader { return io.LimitReader(newRandReader(rand.New(rand.NewSource(seed))), size) } +type fakeTree struct { + t testing.TB + repo *repository.Repository + knownBlobs backend.IDSet +} + // saveFile reads from rd and saves the blobs in the repository. The list of // IDs is returned. -func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs backend.IDs) { - ch := chunker.New(rd, repo.Config.ChunkerPolynomial) +func (f fakeTree) saveFile(rd io.Reader) (blobs backend.IDs) { + blobs = backend.IDs{} + ch := chunker.New(rd, f.repo.Config.ChunkerPolynomial) for { chunk, err := ch.Next(getBuf()) @@ -101,47 +109,92 @@ func saveFile(t testing.TB, repo *repository.Repository, rd io.Reader) (blobs ba } if err != nil { - t.Fatalf("unabel to save chunk in repo: %v", err) + f.t.Fatalf("unable to save chunk in repo: %v", err) } - id, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil) - if err != nil { - t.Fatalf("error saving chunk: %v", err) + id := backend.Hash(chunk.Data) + if !f.knownBlobs.Has(id) { + _, err := f.repo.SaveAndEncrypt(pack.Data, chunk.Data, &id) + if err != nil { + f.t.Fatalf("error saving chunk: %v", err) + } + f.knownBlobs.Insert(id) } + blobs = append(blobs, id) } return blobs } -const maxFileSize = 1500000 -const maxSeed = 100 +const ( + maxFileSize = 1500000 + maxSeed = 32 + maxNodes = 32 +) -// saveTree saves a tree of fake files in the repo and returns the ID. -func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID { +func (f fakeTree) treeIsKnown(tree *Tree) (bool, backend.ID) { + data, err := json.Marshal(tree) + if err != nil { + f.t.Fatalf("json.Marshal(tree) returned error: %v", err) + return false, backend.ID{} + } + data = append(data, '\n') + + // check if tree has been saved before + id := backend.Hash(data) + if f.knownBlobs.Has(id) { + return true, id + } + + return false, id +} + +// save stores a tree of fake files in the repo and returns the ID. +func (f fakeTree) saveTree(seed int64, depth int) backend.ID { rnd := rand.NewSource(seed) - numNodes := int(rnd.Int63() % 64) - t.Logf("create %v nodes", numNodes) + numNodes := int(rnd.Int63() % maxNodes) var tree Tree for i := 0; i < numNodes; i++ { - seed := rnd.Int63() % maxSeed - size := rnd.Int63() % maxFileSize - node := &Node{ - Name: fmt.Sprintf("file-%v", seed), - Type: "file", - Mode: 0644, - Size: uint64(size), + // randomly select the type of the node, either tree (p = 1/4) or file (p = 3/4). + if depth > 1 && rnd.Int63()%4 == 0 { + treeSeed := rnd.Int63() % maxSeed + id := f.saveTree(treeSeed, depth-1) + + node := &Node{ + Name: fmt.Sprintf("dir-%v", treeSeed), + Type: "dir", + Mode: 0755, + Subtree: &id, + } + + tree.Nodes = append(tree.Nodes, node) + continue } - node.Content = saveFile(t, repo, fakeFile(t, seed, size)) + fileSeed := rnd.Int63() % maxSeed + fileSize := (maxFileSize / maxSeed) * fileSeed + + node := &Node{ + Name: fmt.Sprintf("file-%v", fileSeed), + Type: "file", + Mode: 0644, + Size: uint64(fileSize), + } + + node.Content = f.saveFile(fakeFile(fileSeed, fileSize)) tree.Nodes = append(tree.Nodes, node) } - id, err := repo.SaveJSON(pack.Tree, tree) + if known, id := f.treeIsKnown(&tree); known { + return id + } + + id, err := f.repo.SaveJSON(pack.Tree, tree) if err != nil { - t.Fatal(err) + f.t.Fatal(err) } return id @@ -149,8 +202,12 @@ func saveTree(t testing.TB, repo *repository.Repository, seed int64) backend.ID // TestCreateSnapshot creates a snapshot filled with fake data. The // fake data is generated deterministically from the timestamp `at`, which is -// also used as the snapshot's timestamp. -func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) backend.ID { +// also used as the snapshot's timestamp. The tree's depth can be specified +// with the parameter depth. +func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time, depth int) *Snapshot { + seed := at.Unix() + t.Logf("create fake snapshot at %s with seed %d", at, seed) + fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05")) snapshot, err := NewSnapshot([]string{fakedir}) if err != nil { @@ -158,7 +215,13 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) } snapshot.Time = at - treeID := saveTree(t, repo, at.UnixNano()) + f := fakeTree{ + t: t, + repo: repo, + knownBlobs: backend.NewIDSet(), + } + + treeID := f.saveTree(seed, depth) snapshot.Tree = &treeID id, err := repo.SaveJSONUnpacked(backend.Snapshot, snapshot) @@ -166,6 +229,8 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) t.Fatal(err) } + snapshot.id = &id + t.Logf("saved snapshot %v", id.Str()) err = repo.Flush() @@ -178,5 +243,5 @@ func TestCreateSnapshot(t testing.TB, repo *repository.Repository, at time.Time) t.Fatal(err) } - return id + return snapshot } diff --git a/src/restic/testing_test.go b/src/restic/testing_test.go index 8243a01a8..fa71038ac 100644 --- a/src/restic/testing_test.go +++ b/src/restic/testing_test.go @@ -10,14 +10,17 @@ import ( var testSnapshotTime = time.Unix(1460289341, 207401672) -const testCreateSnapshots = 3 +const ( + testCreateSnapshots = 3 + testDepth = 2 +) func TestCreateSnapshot(t *testing.T) { repo, cleanup := repository.TestRepository(t) defer cleanup() for i := 0; i < testCreateSnapshots; i++ { - restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second)) + restic.TestCreateSnapshot(t, repo, testSnapshotTime.Add(time.Duration(i)*time.Second), testDepth) } snapshots, err := restic.LoadAllSnapshots(repo)