mirror
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

restore: Rename things, match the rest of the project

This commit is contained in:
Alexander Neumann 2018-07-11 21:33:36 +02:00
parent c021ad2334
commit 941202c119
3 changed files with 113 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
internal/restorer/filerestorer_test.go
View File

@ -13,21 +13,17 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
/////////////////////////////////////////////////////////////////////////////// type TestBlob struct {
// test helpers (TODO move to a dedicated file?)
///////////////////////////////////////////////////////////////////////////////
type _Blob struct {
data string data string
pack string pack string
} }
type _File struct { type TestFile struct {
name string name string
blobs []_Blob blobs []TestBlob
} }
type _TestData struct { type TestRepo struct {
key *crypto.Key key *crypto.Key
// pack names and ids // pack names and ids
@ -47,39 +43,40 @@ type _TestData struct {
idx filePackTraverser idx filePackTraverser
} }
func (i *_TestData) Lookup(blobID restic.ID, _ restic.BlobType) ([]restic.PackedBlob, bool) { func (i *TestRepo) Lookup(blobID restic.ID, _ restic.BlobType) ([]restic.PackedBlob, bool) {
packs, found := i.blobs[blobID] packs, found := i.blobs[blobID]
return packs, found return packs, found
} }
func (i *_TestData) packName(pack *packInfo) string { func (i *TestRepo) packName(pack *packInfo) string {
return i.packsIDToName[pack.id] return i.packsIDToName[pack.id]
} }
func (i *_TestData) packID(name string) restic.ID {
func (i *TestRepo) packID(name string) restic.ID {
return i.packsNameToID[name] return i.packsNameToID[name]
} }
func (i *_TestData) pack(queue *packQueue, name string) *packInfo { func (i *TestRepo) pack(queue *packQueue, name string) *packInfo {
id := i.packsNameToID[name] id := i.packsNameToID[name]
return queue.packs[id] return queue.packs[id]
} }
func (i *_TestData) fileContent(file *fileInfo) string { func (i *TestRepo) fileContent(file *fileInfo) string {
return i.filesPathToContent[file.path] return i.filesPathToContent[file.path]
} }
func _newTestData(_files []_File) *_TestData { func newTestRepo(content []TestFile) *TestRepo {
type _Pack struct { type Pack struct {
name string name string
data []byte data []byte
blobs map[restic.ID]restic.Blob blobs map[restic.ID]restic.Blob
} }
_packs := make(map[string]_Pack) packs := make(map[string]Pack)
key := crypto.NewRandomKey() key := crypto.NewRandomKey()
seal := func(data []byte) []byte { seal := func(data []byte) []byte {
ciphertext := restic.NewBlobBuffer(len(data)) ciphertext := restic.NewBlobBuffer(len(data))
ciphertext = ciphertext[:0] // TODO what does this actually do? ciphertext = ciphertext[:0] // truncate the slice
nonce := crypto.NewRandomNonce() nonce := crypto.NewRandomNonce()
ciphertext = append(ciphertext, nonce...) ciphertext = append(ciphertext, nonce...)
return key.Seal(ciphertext, nonce, data, nil) return key.Seal(ciphertext, nonce, data, nil)
@ -87,34 +84,34 @@ func _newTestData(_files []_File) *_TestData {
filesPathToContent := make(map[string]string) filesPathToContent := make(map[string]string)
for _, _file := range _files { for _, file := range content {
var content string var content string
for _, _blob := range _file.blobs { for _, blob := range file.blobs {
content += _blob.data content += blob.data
// get the pack, create as necessary // get the pack, create as necessary
var _pack _Pack var pack Pack
var found bool // TODO is there more concise way of doing this in go? var found bool
if _pack, found = _packs[_blob.pack]; !found { if pack, found = packs[blob.pack]; !found {
_pack = _Pack{name: _blob.pack, blobs: make(map[restic.ID]restic.Blob)} pack = Pack{name: blob.pack, blobs: make(map[restic.ID]restic.Blob)}
} }
// calculate blob id and add to the pack as necessary // calculate blob id and add to the pack as necessary
_blobID := restic.Hash([]byte(_blob.data)) blobID := restic.Hash([]byte(blob.data))
if _, found := _pack.blobs[_blobID]; !found { if _, found := pack.blobs[blobID]; !found {
_blobData := seal([]byte(_blob.data)) blobData := seal([]byte(blob.data))
_pack.blobs[_blobID] = restic.Blob{ pack.blobs[blobID] = restic.Blob{
Type: restic.DataBlob, Type: restic.DataBlob,
ID: _blobID, ID: blobID,
Length: uint(len(_blobData)), // XXX is Length encrypted or plaintext? Length: uint(len(blobData)),
Offset: uint(len(_pack.data)), Offset: uint(len(pack.data)),
} }
_pack.data = append(_pack.data, _blobData...) pack.data = append(pack.data, blobData...)
} }
_packs[_blob.pack] = _pack packs[blob.pack] = pack
} }
filesPathToContent[_file.name] = content filesPathToContent[file.name] = content
} }
blobs := make(map[restic.ID][]restic.PackedBlob) blobs := make(map[restic.ID][]restic.PackedBlob)
@ -122,26 +119,26 @@ func _newTestData(_files []_File) *_TestData {
packsIDToData := make(map[restic.ID][]byte) packsIDToData := make(map[restic.ID][]byte)
packsNameToID := make(map[string]restic.ID) packsNameToID := make(map[string]restic.ID)
for _, _pack := range _packs { for _, pack := range packs {
_packID := restic.Hash(_pack.data) packID := restic.Hash(pack.data)
packsIDToName[_packID] = _pack.name packsIDToName[packID] = pack.name
packsIDToData[_packID] = _pack.data packsIDToData[packID] = pack.data
packsNameToID[_pack.name] = _packID packsNameToID[pack.name] = packID
for blobID, blob := range _pack.blobs { for blobID, blob := range pack.blobs {
blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: _packID}) blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID})
} }
} }
var files []*fileInfo var files []*fileInfo
for _, _file := range _files { for _, file := range content {
content := restic.IDs{} content := restic.IDs{}
for _, _blob := range _file.blobs { for _, blob := range file.blobs {
content = append(content, restic.Hash([]byte(_blob.data))) content = append(content, restic.Hash([]byte(blob.data)))
} }
files = append(files, &fileInfo{path: _file.name, blobs: content}) files = append(files, &fileInfo{path: file.name, blobs: content})
} }
_data := &_TestData{ repo := &TestRepo{
key: key, key: key,
packsIDToName: packsIDToName, packsIDToName: packsIDToName,
packsIDToData: packsIDToData, packsIDToData: packsIDToData,
@ -150,30 +147,30 @@ func _newTestData(_files []_File) *_TestData {
files: files, files: files,
filesPathToContent: filesPathToContent, filesPathToContent: filesPathToContent,
} }
_data.idx = filePackTraverser{lookup: _data.Lookup} repo.idx = filePackTraverser{lookup: repo.Lookup}
_data.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
packID, err := restic.ParseID(h.Name) packID, err := restic.ParseID(h.Name)
if err != nil { if err != nil {
return err return err
} }
rd := bytes.NewReader(_data.packsIDToData[packID][int(offset) : int(offset)+length]) rd := bytes.NewReader(repo.packsIDToData[packID][int(offset) : int(offset)+length])
return fn(rd) return fn(rd)
} }
return _data return repo
} }
func restoreAndVerify(t *testing.T, _files []_File) { func restoreAndVerify(t *testing.T, content []TestFile) {
test := _newTestData(_files) repo := newTestRepo(content)
r := newFileRestorer(test.loader, test.key, test.idx) r := newFileRestorer(repo.loader, repo.key, repo.idx)
r.files = test.files r.files = repo.files
r.restoreFiles(context.TODO(), func(path string, err error) { r.restoreFiles(context.TODO(), func(path string, err error) {
rtest.OK(t, errors.Wrapf(err, "unexpected error")) rtest.OK(t, errors.Wrapf(err, "unexpected error"))
}) })
for _, file := range test.files { for _, file := range repo.files {
data, err := ioutil.ReadFile(file.path) data, err := ioutil.ReadFile(file.path)
if err != nil { if err != nil {
t.Errorf("unable to read file %v: %v", file.path, err) t.Errorf("unable to read file %v: %v", file.path, err)
@ -182,7 +179,7 @@ func restoreAndVerify(t *testing.T, _files []_File) {
rtest.Equals(t, false, r.filesWriter.writers.Contains(file.path)) rtest.Equals(t, false, r.filesWriter.writers.Contains(file.path))
content := test.fileContent(file) content := repo.fileContent(file)
if !bytes.Equal(data, []byte(content)) { if !bytes.Equal(data, []byte(content)) {
t.Errorf("file %v has wrong content: want %q, got %q", file.path, content, data) t.Errorf("file %v has wrong content: want %q, got %q", file.path, content, data)
} }
@ -191,36 +188,36 @@ func restoreAndVerify(t *testing.T, _files []_File) {
rtest.OK(t, nil) rtest.OK(t, nil)
} }
func TestFileRestorer_basic(t *testing.T) { func TestFileRestorerBasic(t *testing.T) {
tempdir, cleanup := rtest.TempDir(t) tempdir, cleanup := rtest.TempDir(t)
defer cleanup() defer cleanup()
restoreAndVerify(t, []_File{ restoreAndVerify(t, []TestFile{
_File{ TestFile{
name: tempdir + "/file1", name: tempdir + "/file1",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1-1", "pack1-1"}, TestBlob{"data1-1", "pack1-1"},
_Blob{"data1-2", "pack1-2"}, TestBlob{"data1-2", "pack1-2"},
}, },
}, },
_File{ TestFile{
name: tempdir + "/file2", name: tempdir + "/file2",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data2-1", "pack2-1"}, TestBlob{"data2-1", "pack2-1"},
_Blob{"data2-2", "pack2-2"}, TestBlob{"data2-2", "pack2-2"},
}, },
}, },
}) })
} }
func TestFileRestorer_emptyFile(t *testing.T) { func TestFileRestorerEmptyFile(t *testing.T) {
tempdir, cleanup := rtest.TempDir(t) tempdir, cleanup := rtest.TempDir(t)
defer cleanup() defer cleanup()
restoreAndVerify(t, []_File{ restoreAndVerify(t, []TestFile{
_File{ TestFile{
name: tempdir + "/empty", name: tempdir + "/empty",
blobs: []_Blob{}, blobs: []TestBlob{},
}, },
}) })
} }

12
internal/restorer/packcache_test.go
View File

@ -39,7 +39,7 @@ func TestBytesWriterSeeker(t *testing.T) {
assertNotOK(t, "unsupported seek", err) assertNotOK(t, "unsupported seek", err)
} }
func TestPackCache_basic(t *testing.T) { func TestPackCacheBasic(t *testing.T) {
assertReader := func(expected []byte, offset int64, rd io.ReaderAt) { assertReader := func(expected []byte, offset int64, rd io.ReaderAt) {
actual := make([]byte, len(expected)) actual := make([]byte, len(expected))
rd.ReadAt(actual, offset) rd.ReadAt(actual, offset)
@ -89,7 +89,7 @@ func TestPackCache_basic(t *testing.T) {
assertReader([]byte{1, 2, 3, 4, 5}, 10, rd) assertReader([]byte{1, 2, 3, 4, 5}, 10, rd)
} }
func TestPackCache_invalid_range(t *testing.T) { func TestPackCacheInvalidRange(t *testing.T) {
c := newPackCache(10) c := newPackCache(10)
id := restic.NewRandomID() id := restic.NewRandomID()
@ -113,7 +113,7 @@ func TestPackCache_invalid_range(t *testing.T) {
assertNotOK(t, "negative length", err) assertNotOK(t, "negative length", err)
} }
func TestPackCache_capacity(t *testing.T) { func TestPackCacheCapacity(t *testing.T) {
c := newPackCache(10) c := newPackCache(10)
id1, id2, id3 := restic.NewRandomID(), restic.NewRandomID(), restic.NewRandomID() id1, id2, id3 := restic.NewRandomID(), restic.NewRandomID(), restic.NewRandomID()
@ -159,7 +159,7 @@ func TestPackCache_capacity(t *testing.T) {
rtest.Equals(t, true, loaded) rtest.Equals(t, true, loaded)
} }
func TestPackCache_downsize_record(t *testing.T) { func TestPackCacheDownsizeRecord(t *testing.T) {
c := newPackCache(10) c := newPackCache(10)
id := restic.NewRandomID() id := restic.NewRandomID()
@ -233,7 +233,7 @@ func TestPackCache_downsize_record(t *testing.T) {
rd.Close() rd.Close()
} }
func TestPackCache_wrong_load_size(t *testing.T) { func TestPackCacheWrongLoadSize(t *testing.T) {
c := newPackCache(10) c := newPackCache(10)
_, err := c.get(restic.NewRandomID(), 0, 5, func(offset int64, length int, wr io.WriteSeeker) error { _, err := c.get(restic.NewRandomID(), 0, 5, func(offset int64, length int, wr io.WriteSeeker) error {
@ -249,7 +249,7 @@ func TestPackCache_wrong_load_size(t *testing.T) {
assertNotOK(t, "too many bytes read", err) assertNotOK(t, "too many bytes read", err)
} }
func TestPackCache_invalidRequests(t *testing.T) { func TestPackCacheInvalidRequests(t *testing.T) {
c := newPackCache(10) c := newPackCache(10)
id := restic.NewRandomID() id := restic.NewRandomID()

80
internal/restorer/packqueue_test.go
View File

@ -7,7 +7,7 @@ import (
rtest "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test"
) )
func processPack(t *testing.T, data *_TestData, pack *packInfo, files []*fileInfo) { func processPack(t *testing.T, data *TestRepo, pack *packInfo, files []*fileInfo) {
for _, file := range files { for _, file := range files {
data.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool { data.idx.forEachFilePack(file, func(packIdx int, packID restic.ID, packBlobs []restic.Blob) bool {
// assert file's head pack // assert file's head pack
@ -18,13 +18,13 @@ func processPack(t *testing.T, data *_TestData, pack *packInfo, files []*fileInf
} }
} }
func TestPackQueue_basic(t *testing.T) { func TestPackQueueBasic(t *testing.T) {
data := _newTestData([]_File{ data := newTestRepo([]TestFile{
_File{ TestFile{
name: "file", name: "file",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1", "pack1"}, TestBlob{"data1", "pack1"},
_Blob{"data2", "pack2"}, TestBlob{"data2", "pack2"},
}, },
}, },
}) })
@ -83,16 +83,16 @@ func TestPackQueue_basic(t *testing.T) {
rtest.Equals(t, true, queue.isEmpty()) rtest.Equals(t, true, queue.isEmpty())
} }
func TestPackQueue_failedFile(t *testing.T) { func TestPackQueueFailedFile(t *testing.T) {
// point of this test is to assert that enqueuePack removes // point of this test is to assert that enqueuePack removes
// all references to files that failed restore // all references to files that failed restore
data := _newTestData([]_File{ data := newTestRepo([]TestFile{
_File{ TestFile{
name: "file", name: "file",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1", "pack1"}, TestBlob{"data1", "pack1"},
_Blob{"data2", "pack2"}, TestBlob{"data2", "pack2"},
}, },
}, },
}) })
@ -105,23 +105,23 @@ func TestPackQueue_failedFile(t *testing.T) {
rtest.Equals(t, true, queue.isEmpty()) rtest.Equals(t, true, queue.isEmpty())
} }
func TestPackQueue_ordering_cost(t *testing.T) { func TestPackQueueOrderingCost(t *testing.T) {
// assert pack1 is selected before pack2: // assert pack1 is selected before pack2:
// pack1 is ready to restore file1, pack2 is ready to restore file2 // pack1 is ready to restore file1, pack2 is ready to restore file2
// but pack2 cannot be immediately used to restore file1 // but pack2 cannot be immediately used to restore file1
data := _newTestData([]_File{ data := newTestRepo([]TestFile{
_File{ TestFile{
name: "file1", name: "file1",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1", "pack1"}, TestBlob{"data1", "pack1"},
_Blob{"data2", "pack2"}, TestBlob{"data2", "pack2"},
}, },
}, },
_File{ TestFile{
name: "file2", name: "file2",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data2", "pack2"}, TestBlob{"data2", "pack2"},
}, },
}, },
}) })
@ -143,22 +143,22 @@ func TestPackQueue_ordering_cost(t *testing.T) {
rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{})) rtest.Equals(t, false, queue.requeuePack(pack, files, []*fileInfo{}))
} }
func TestPackQueue_ordering_inprogress(t *testing.T) { func TestPackQueueOrderingInprogress(t *testing.T) {
// finish restoring one file before starting another // finish restoring one file before starting another
data := _newTestData([]_File{ data := newTestRepo([]TestFile{
_File{ TestFile{
name: "file1", name: "file1",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1-1", "pack1-1"}, TestBlob{"data1-1", "pack1-1"},
_Blob{"data1-2", "pack1-2"}, TestBlob{"data1-2", "pack1-2"},
}, },
}, },
_File{ TestFile{
name: "file2", name: "file2",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data2-1", "pack2-1"}, TestBlob{"data2-1", "pack2-1"},
_Blob{"data2-2", "pack2-2"}, TestBlob{"data2-2", "pack2-2"},
}, },
}, },
}) })
@ -192,17 +192,17 @@ func TestPackQueue_ordering_inprogress(t *testing.T) {
rtest.Equals(t, false, file == files[0]) // different file as before rtest.Equals(t, false, file == files[0]) // different file as before
} }
func TestPackQueue_packMultiuse(t *testing.T) { func TestPackQueuePackMultiuse(t *testing.T) {
// the same pack is required multiple times to restore the same file // the same pack is required multiple times to restore the same file
data := _newTestData([]_File{ data := newTestRepo([]TestFile{
_File{ TestFile{
name: "file", name: "file",
blobs: []_Blob{ blobs: []TestBlob{
_Blob{"data1", "pack1"}, TestBlob{"data1", "pack1"},
_Blob{"data2", "pack2"}, TestBlob{"data2", "pack2"},
_Blob{"data3", "pack1"}, // pack1 reuse, new blob TestBlob{"data3", "pack1"}, // pack1 reuse, new blob
_Blob{"data2", "pack2"}, // pack2 reuse, same blob TestBlob{"data2", "pack2"}, // pack2 reuse, same blob
}, },
}, },
}) })