From 5b601f00b13996d5dacd5b7358ff6687341ac1dd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 18 Nov 2015 20:33:20 +0100 Subject: [PATCH 01/13] Add error checking --- repository/index.go | 6 +++--- repository/index_test.go | 2 +- repository/key.go | 10 +++++----- repository/repository.go | 22 ++++++++++++++++------ 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/repository/index.go b/repository/index.go index b75d69992..7bb4a273e 100644 --- a/repository/index.go +++ b/repository/index.go @@ -556,7 +556,7 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) { } // LoadIndexWithDecoder loads the index and decodes it with fn. -func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (*Index, error) { +func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8]) idxID, err := backend.ParseID(id) @@ -568,9 +568,9 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde if err != nil { return nil, err } - defer rd.Close() + defer closeOrErr(rd, &err) - idx, err := fn(rd) + idx, err = fn(rd) if err != nil { debug.Log("LoadIndexWithDecoder", "error while decoding index %v: %v", id, err) return nil, err diff --git a/repository/index_test.go b/repository/index_test.go index 4a6f270fd..2e8a51e84 100644 --- a/repository/index_test.go +++ b/repository/index_test.go @@ -129,7 +129,7 @@ func TestIndexSerialize(t *testing.T) { "index not final after encoding") id := randomID() - idx.SetID(id) + OK(t, idx.SetID(id)) id2, err := idx.ID() Assert(t, id2.Equal(id), "wrong ID returned: want %v, got %v", id, id2) diff --git a/repository/key.go b/repository/key.go index 294c8561b..40e75f11d 100644 --- a/repository/key.go +++ b/repository/key.go @@ -112,23 +112,23 @@ func SearchKey(s *Repository, password string) (*Key, error) { } // LoadKey loads a key from the backend. -func LoadKey(s *Repository, name string) (*Key, error) { +func LoadKey(s *Repository, name string) (k *Key, err error) { // extract data from repo rd, err := s.be.Get(backend.Key, name) if err != nil { return nil, err } - defer rd.Close() + defer closeOrErr(rd, &err) // restore json dec := json.NewDecoder(rd) - k := Key{} - err = dec.Decode(&k) + k = new(Key) + err = dec.Decode(k) if err != nil { return nil, err } - return &k, nil + return k, nil } // AddKey adds a new key to an already existing repository. diff --git a/repository/repository.go b/repository/repository.go index 2aeceae54..03ba4ed59 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -143,19 +143,29 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byt return plaintextBuf, nil } +// closeOrErr calls cl.Close() and sets err to the returned error value if +// itself is not yet set. +func closeOrErr(cl io.Closer, err *error) { + e := cl.Close() + if *err != nil { + return + } + *err = e +} + // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on // the item. -func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error { +func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) (err error) { // load blob from backend rd, err := r.be.Get(t, id.String()) if err != nil { return err } - defer rd.Close() + defer closeOrErr(rd, &err) // decrypt decryptRd, err := crypto.DecryptFrom(r.key, rd) - defer decryptRd.Close() + defer closeOrErr(decryptRd, &err) if err != nil { return err } @@ -172,7 +182,7 @@ func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interf // LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the // data and afterwards call json.Unmarshal on the item. -func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) error { +func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) (err error) { // lookup pack blob, err := r.idx.Lookup(id) if err != nil { @@ -184,11 +194,11 @@ func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface if err != nil { return err } - defer rd.Close() + defer closeOrErr(rd, &err) // decrypt decryptRd, err := crypto.DecryptFrom(r.key, rd) - defer decryptRd.Close() + defer closeOrErr(decryptRd, &err) if err != nil { return err } From d4b873ca76ec545e8f2e0be37ba7dd5d0cd33076 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 21 Nov 2015 16:45:05 +0100 Subject: [PATCH 02/13] Repo: add documentation --- repository/doc.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/repository/doc.go b/repository/doc.go index ac8463066..cb98334c4 100644 --- a/repository/doc.go +++ b/repository/doc.go @@ -1,2 +1,28 @@ -// Package repository implements a restic repository on top of a backend. +// Package repository implements a restic repository on top of a backend. In +// the following the abstractions used for this package are listed. More +// information can be found in the restic design document. +// +// File +// +// A file is a named handle for some data saved in the backend. For the local +// backend, this corresponds to actual files saved to disk. Usually, the SHA256 +// hash of the content is used for a file's name (hexadecimal, in lower-case +// ASCII characters). An exception is the file `config`. Most files are +// encrypted before being saved in a backend. This means that the name is the +// hash of the ciphertext. +// +// Blob +// +// A blob is a number of bytes that has a type (data or tree). Blobs are +// identified by an ID, which is the SHA256 hash of the blobs' contents. One or +// more blobs are bundled together in a Pack and then saved to the backend. +// Blobs are always encrypted before being bundled in a Pack. +// +// Pack +// +// A Pack is a File in the backend that contains one or more (encrypted) blobs, +// followed by a header at the end of the Pack. The header is encrypted and +// contains the ID, type, length and offset for each blob contained in the +// Pack. +// package repository From 8209bb309bf061614b9db8ec1d48486b78094b6f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 18 Nov 2015 20:20:25 +0100 Subject: [PATCH 03/13] split out decryptReader and packerManager --- repository/decrypt_read_closer.go | 36 +++++++++ repository/packer_manager.go | 102 +++++++++++++++++++++++++ repository/repository.go | 123 ++---------------------------- 3 files changed, 144 insertions(+), 117 deletions(-) create mode 100644 repository/decrypt_read_closer.go create mode 100644 repository/packer_manager.go diff --git a/repository/decrypt_read_closer.go b/repository/decrypt_read_closer.go new file mode 100644 index 000000000..81a6f9513 --- /dev/null +++ b/repository/decrypt_read_closer.go @@ -0,0 +1,36 @@ +package repository + +import ( + "io" + + "github.com/restic/restic/crypto" +) + +// decryptReadCloser couples an underlying reader with a DecryptReader and +// implements io.ReadCloser. On Close(), both readers are closed. +type decryptReadCloser struct { + r io.ReadCloser + dr io.ReadCloser +} + +func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) { + dr, err := crypto.DecryptFrom(key, rd) + if err != nil { + return nil, err + } + + return &decryptReadCloser{r: rd, dr: dr}, nil +} + +func (dr *decryptReadCloser) Read(buf []byte) (int, error) { + return dr.dr.Read(buf) +} + +func (dr *decryptReadCloser) Close() error { + err := dr.dr.Close() + if err != nil { + return err + } + + return dr.r.Close() +} diff --git a/repository/packer_manager.go b/repository/packer_manager.go new file mode 100644 index 000000000..99b74cea4 --- /dev/null +++ b/repository/packer_manager.go @@ -0,0 +1,102 @@ +package repository + +import ( + "sync" + + "github.com/restic/chunker" + "github.com/restic/restic/backend" + "github.com/restic/restic/crypto" + "github.com/restic/restic/debug" + "github.com/restic/restic/pack" +) + +// packerManager keeps a list of open packs and creates new on demand. +type packerManager struct { + be backend.Backend + key *crypto.Key + pm sync.Mutex + packs []*pack.Packer +} + +const minPackSize = 4 * chunker.MiB +const maxPackSize = 16 * chunker.MiB +const maxPackers = 200 + +// findPacker returns a packer for a new blob of size bytes. Either a new one is +// created or one is returned that already has some blobs. +func (r *packerManager) findPacker(size uint) (*pack.Packer, error) { + r.pm.Lock() + defer r.pm.Unlock() + + // search for a suitable packer + if len(r.packs) > 0 { + debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size) + for i, p := range r.packs { + if p.Size()+size < maxPackSize { + debug.Log("Repo.findPacker", "found packer %v", p) + // remove from list + r.packs = append(r.packs[:i], r.packs[i+1:]...) + return p, nil + } + } + } + + // no suitable packer found, return new + blob, err := r.be.Create() + if err != nil { + return nil, err + } + debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size) + return pack.NewPacker(r.key, blob), nil +} + +// insertPacker appends p to s.packs. +func (r *packerManager) insertPacker(p *pack.Packer) { + r.pm.Lock() + defer r.pm.Unlock() + + r.packs = append(r.packs, p) + debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs)) +} + +// savePacker stores p in the backend. +func (r *Repository) savePacker(p *pack.Packer) error { + debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count()) + _, err := p.Finalize() + if err != nil { + return err + } + + // move file to the final location + sid := p.ID() + err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String()) + if err != nil { + debug.Log("Repo.savePacker", "blob Finalize() error: %v", err) + return err + } + + debug.Log("Repo.savePacker", "saved as %v", sid.Str()) + + // update blobs in the index + for _, b := range p.Blobs() { + debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), sid.Str()) + r.idx.Current().Store(PackedBlob{ + Type: b.Type, + ID: b.ID, + PackID: sid, + Offset: b.Offset, + Length: uint(b.Length), + }) + r.idx.RemoveFromInFlight(b.ID) + } + + return nil +} + +// countPacker returns the number of open (unfinished) packers. +func (r *packerManager) countPacker() int { + r.pm.Lock() + defer r.pm.Unlock() + + return len(r.packs) +} diff --git a/repository/repository.go b/repository/repository.go index 03ba4ed59..0b1955486 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -9,9 +9,7 @@ import ( "io" "io/ioutil" "os" - "sync" - "github.com/restic/chunker" "github.com/restic/restic/backend" "github.com/restic/restic/crypto" "github.com/restic/restic/debug" @@ -26,8 +24,7 @@ type Repository struct { keyName string idx *MasterIndex - pm sync.Mutex - packs []*pack.Packer + *packerManager } // New returns a new repository with backend be. @@ -35,6 +32,9 @@ func New(be backend.Backend) *Repository { return &Repository{ be: be, idx: NewMasterIndex(), + packerManager: &packerManager{ + be: be, + }, } } @@ -218,90 +218,6 @@ func (r *Repository) LookupBlobSize(id backend.ID) (uint, error) { return r.idx.LookupSize(id) } -const minPackSize = 4 * chunker.MiB -const maxPackSize = 16 * chunker.MiB -const maxPackers = 200 - -// findPacker returns a packer for a new blob of size bytes. Either a new one is -// created or one is returned that already has some blobs. -func (r *Repository) findPacker(size uint) (*pack.Packer, error) { - r.pm.Lock() - defer r.pm.Unlock() - - // search for a suitable packer - if len(r.packs) > 0 { - debug.Log("Repo.findPacker", "searching packer for %d bytes\n", size) - for i, p := range r.packs { - if p.Size()+size < maxPackSize { - debug.Log("Repo.findPacker", "found packer %v", p) - // remove from list - r.packs = append(r.packs[:i], r.packs[i+1:]...) - return p, nil - } - } - } - - // no suitable packer found, return new - blob, err := r.be.Create() - if err != nil { - return nil, err - } - debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size) - return pack.NewPacker(r.key, blob), nil -} - -// insertPacker appends p to s.packs. -func (r *Repository) insertPacker(p *pack.Packer) { - r.pm.Lock() - defer r.pm.Unlock() - - r.packs = append(r.packs, p) - debug.Log("Repo.insertPacker", "%d packers\n", len(r.packs)) -} - -// savePacker stores p in the backend. -func (r *Repository) savePacker(p *pack.Packer) error { - debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count()) - _, err := p.Finalize() - if err != nil { - return err - } - - // move file to the final location - sid := p.ID() - err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String()) - if err != nil { - debug.Log("Repo.savePacker", "blob Finalize() error: %v", err) - return err - } - - debug.Log("Repo.savePacker", "saved as %v", sid.Str()) - - // update blobs in the index - var packedBlobs []PackedBlob - for _, b := range p.Blobs() { - packedBlobs = append(packedBlobs, PackedBlob{ - Type: b.Type, - ID: b.ID, - PackID: sid, - Offset: b.Offset, - Length: uint(b.Length), - }) - r.idx.RemoveFromInFlight(b.ID) - } - r.idx.Current().StoreBlobs(packedBlobs) - - return nil -} - -// countPacker returns the number of open (unfinished) packers. -func (r *Repository) countPacker() int { - r.pm.Lock() - defer r.pm.Unlock() - - return len(r.packs) -} - // SaveAndEncrypt encrypts data and stores it to the backend as type t. If data is small // enough, it will be packed together with other small blobs. func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID) (backend.ID, error) { @@ -628,35 +544,6 @@ func LoadIndex(repo *Repository, id string) (*Index, error) { return nil, err } -// decryptReadCloser couples an underlying reader with a DecryptReader and -// implements io.ReadCloser. On Close(), both readers are closed. -type decryptReadCloser struct { - r io.ReadCloser - dr io.ReadCloser -} - -func newDecryptReadCloser(key *crypto.Key, rd io.ReadCloser) (io.ReadCloser, error) { - dr, err := crypto.DecryptFrom(key, rd) - if err != nil { - return nil, err - } - - return &decryptReadCloser{r: rd, dr: dr}, nil -} - -func (dr *decryptReadCloser) Read(buf []byte) (int, error) { - return dr.dr.Read(buf) -} - -func (dr *decryptReadCloser) Close() error { - err := dr.dr.Close() - if err != nil { - return err - } - - return dr.r.Close() -} - // GetDecryptReader opens the file id stored in the backend and returns a // reader that yields the decrypted content. The reader must be closed. func (r *Repository) GetDecryptReader(t backend.Type, id string) (io.ReadCloser, error) { @@ -677,6 +564,7 @@ func (r *Repository) SearchKey(password string) error { } r.key = key.master + r.packerManager.key = key.master r.keyName = key.Name() r.Config, err = LoadConfig(r) return err @@ -699,6 +587,7 @@ func (r *Repository) Init(password string) error { } r.key = key.master + r.packerManager.key = key.master r.keyName = key.Name() r.Config, err = CreateConfig(r) return err From 4fd7676e92f160492e8781adc5c1ced94ab282f1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 29 Nov 2015 14:27:52 +0100 Subject: [PATCH 04/13] HashingWriter: Add documentation --- backend/writer.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/writer.go b/backend/writer.go index ae2c5bfd9..6bee58b49 100644 --- a/backend/writer.go +++ b/backend/writer.go @@ -46,12 +46,14 @@ func (h *HashAppendWriter) Write(p []byte) (n int, err error) { return 0, errors.New("Write() called on closed HashAppendWriter") } +// HashingWriter wraps an io.Writer to hashes all data that is written to it. type HashingWriter struct { w io.Writer h hash.Hash size int } +// NewHashAppendWriter wraps the writer w and feeds all data written to the hash h. func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter { return &HashingWriter{ h: h, @@ -59,16 +61,19 @@ func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter { } } +// Write wraps the write method of the underlying writer and also hashes all data. func (h *HashingWriter) Write(p []byte) (int, error) { n, err := h.w.Write(p) h.size += n return n, err } +// Sum returns the hash of all data written so far. func (h *HashingWriter) Sum(d []byte) []byte { return h.h.Sum(d) } +// Size returns the number of bytes written to the underlying writer. func (h *HashingWriter) Size() int { return h.size } From 0d5731383f756eb2308cd06435d78009e3f43af9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 21 Nov 2015 17:23:59 +0100 Subject: [PATCH 05/13] Remove HashAppendWriter --- backend/writer.go | 41 ----------------------------------------- backend/writer_test.go | 36 ------------------------------------ 2 files changed, 77 deletions(-) diff --git a/backend/writer.go b/backend/writer.go index 6bee58b49..5764b872c 100644 --- a/backend/writer.go +++ b/backend/writer.go @@ -1,51 +1,10 @@ package backend import ( - "errors" "hash" "io" ) -type HashAppendWriter struct { - w io.Writer - origWr io.Writer - h hash.Hash - sum []byte - closed bool -} - -func NewHashAppendWriter(w io.Writer, h hash.Hash) *HashAppendWriter { - return &HashAppendWriter{ - h: h, - w: io.MultiWriter(w, h), - origWr: w, - sum: make([]byte, 0, h.Size()), - } -} - -func (h *HashAppendWriter) Close() error { - if h == nil { - return nil - } - - if !h.closed { - h.closed = true - - _, err := h.origWr.Write(h.h.Sum(nil)) - return err - } - - return nil -} - -func (h *HashAppendWriter) Write(p []byte) (n int, err error) { - if !h.closed { - return h.w.Write(p) - } - - return 0, errors.New("Write() called on closed HashAppendWriter") -} - // HashingWriter wraps an io.Writer to hashes all data that is written to it. type HashingWriter struct { w io.Writer diff --git a/backend/writer_test.go b/backend/writer_test.go index 816eb401a..9fda2c06f 100644 --- a/backend/writer_test.go +++ b/backend/writer_test.go @@ -12,42 +12,6 @@ import ( . "github.com/restic/restic/test" ) -func TestHashAppendWriter(t *testing.T) { - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(rand.Reader, data) - if err != nil { - t.Fatalf("ReadFull: %v", err) - } - - expectedHash := sha256.Sum256(data) - - target := bytes.NewBuffer(nil) - wr := backend.NewHashAppendWriter(target, sha256.New()) - - _, err = wr.Write(data) - OK(t, err) - OK(t, wr.Close()) - - Assert(t, len(target.Bytes()) == size+len(expectedHash), - "HashAppendWriter: invalid number of bytes written: got %d, expected %d", - len(target.Bytes()), size+len(expectedHash)) - - r := target.Bytes() - resultingHash := r[len(r)-len(expectedHash):] - Assert(t, bytes.Equal(expectedHash[:], resultingHash), - "HashAppendWriter: hashes do not match: expected %02x, got %02x", - expectedHash, resultingHash) - - // write again, this must return an error - _, err = wr.Write([]byte{23}) - Assert(t, err != nil, - "HashAppendWriter: Write() after Close() did not return an error") - } -} - func TestHashingWriter(t *testing.T) { tests := []int{5, 23, 2<<18 + 23, 1 << 20} From da71da23d934b4c911b0aa29eafe2565f303529e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 15:15:07 +0100 Subject: [PATCH 06/13] Add MockBackend --- backend/mock_backend.go | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 backend/mock_backend.go diff --git a/backend/mock_backend.go b/backend/mock_backend.go new file mode 100644 index 000000000..92d5521c7 --- /dev/null +++ b/backend/mock_backend.go @@ -0,0 +1,97 @@ +package backend + +import ( + "errors" + "io" +) + +// MockBackend implements a backend whose functions can be specified. This +// should only be used for tests. +type MockBackend struct { + CloseFn func() error + CreateFn func() (Blob, error) + GetFn func(Type, string) (io.ReadCloser, error) + GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) + ListFn func(Type, <-chan struct{}) <-chan string + RemoveFn func(Type, string) error + TestFn func(Type, string) (bool, error) + DeleteFn func() error + LocationFn func() string +} + +func (m *MockBackend) Close() error { + if m.CloseFn == nil { + return nil + } + + return m.CloseFn() +} + +func (m *MockBackend) Location() string { + if m.LocationFn == nil { + return "" + } + + return m.LocationFn() +} + +func (m *MockBackend) Create() (Blob, error) { + if m.CreateFn == nil { + return nil, errors.New("not implemented") + } + + return m.CreateFn() +} + +func (m *MockBackend) Get(t Type, name string) (io.ReadCloser, error) { + if m.GetFn == nil { + return nil, errors.New("not implemented") + } + + return m.GetFn(t, name) +} + +func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { + if m.GetReaderFn == nil { + return nil, errors.New("not implemented") + } + + return m.GetReaderFn(t, name, offset, len) +} + +func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string { + if m.ListFn == nil { + ch := make(chan string) + close(ch) + return ch + } + + return m.ListFn(t, done) +} + +func (m *MockBackend) Remove(t Type, name string) error { + if m.RemoveFn == nil { + return errors.New("not implemented") + } + + return m.RemoveFn(t, name) +} + +func (m *MockBackend) Test(t Type, name string) (bool, error) { + if m.TestFn == nil { + return false, errors.New("not implemented") + } + + return m.TestFn(t, name) +} + +func (m *MockBackend) Delete() error { + if m.DeleteFn == nil { + return errors.New("not implemented") + } + + return m.DeleteFn() +} + +// Make sure that MockBackend implements the backend interface. +var _ Backend = &MockBackend{} From 9cb4e14327bddaf0e8a01480c70ab58aec722a95 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 16:12:00 +0100 Subject: [PATCH 07/13] add MemBackend and MockBackend --- backend/backend_test.go | 5 + backend/mem_backend.go | 192 ++++++++++++++++++++++++++++++++++++ backend/mem_backend_test.go | 12 +++ 3 files changed, 209 insertions(+) create mode 100644 backend/mem_backend.go create mode 100644 backend/mem_backend_test.go diff --git a/backend/backend_test.go b/backend/backend_test.go index 0ff5f01b4..ef928cf8e 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -96,6 +96,11 @@ func testBackend(b backend.Backend, t *testing.T) { err = b.Remove(tpe, test.id) OK(t, err) + // test that the blob is gone + ok, err := b.Test(tpe, test.id) + OK(t, err) + Assert(t, ok == false, "removed blob still present") + // create blob blob, err = b.Create() OK(t, err) diff --git a/backend/mem_backend.go b/backend/mem_backend.go new file mode 100644 index 000000000..213ddb68f --- /dev/null +++ b/backend/mem_backend.go @@ -0,0 +1,192 @@ +package backend + +import ( + "bytes" + "errors" + "io" + "sort" + "sync" +) + +type entry struct { + Type Type + Name string +} + +type memMap map[entry][]byte + +// MemoryBackend is a mock backend that uses a map for storing all data in +// memory. This should only be used for tests. +type MemoryBackend struct { + data memMap + m sync.Mutex + + MockBackend +} + +// NewMemoryBackend returns a new backend that saves all data in a map in +// memory. +func NewMemoryBackend() *MemoryBackend { + be := &MemoryBackend{ + data: make(memMap), + } + + be.MockBackend.TestFn = func(t Type, name string) (bool, error) { + return memTest(be, t, name) + } + + be.MockBackend.CreateFn = func() (Blob, error) { + return memCreate(be) + } + + be.MockBackend.GetFn = func(t Type, name string) (io.ReadCloser, error) { + return memGet(be, t, name) + } + + be.MockBackend.GetReaderFn = func(t Type, name string, offset, length uint) (io.ReadCloser, error) { + return memGetReader(be, t, name, offset, length) + } + + be.MockBackend.RemoveFn = func(t Type, name string) error { + return memRemove(be, t, name) + } + + be.MockBackend.ListFn = func(t Type, done <-chan struct{}) <-chan string { + return memList(be, t, done) + } + + return be +} + +func (be *MemoryBackend) insert(t Type, name string, data []byte) error { + be.m.Lock() + defer be.m.Unlock() + + if _, ok := be.data[entry{t, name}]; ok { + return errors.New("already present") + } + + be.data[entry{t, name}] = data + return nil +} + +func memTest(be *MemoryBackend, t Type, name string) (bool, error) { + be.m.Lock() + defer be.m.Unlock() + + if _, ok := be.data[entry{t, name}]; ok { + return true, nil + } + + return false, nil +} + +// tempMemEntry temporarily holds data written to the memory backend before it +// is finalized. +type tempMemEntry struct { + be *MemoryBackend + data bytes.Buffer +} + +func (e *tempMemEntry) Write(p []byte) (int, error) { + return e.data.Write(p) +} + +func (e *tempMemEntry) Size() uint { + return uint(len(e.data.Bytes())) +} + +func (e *tempMemEntry) Finalize(t Type, name string) error { + return e.be.insert(t, name, e.data.Bytes()) +} + +func memCreate(be *MemoryBackend) (Blob, error) { + return &tempMemEntry{be: be}, nil +} + +// readCloser wraps a reader and adds a noop Close method. +type readCloser struct { + io.Reader +} + +func (rd readCloser) Close() error { + return nil +} + +func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) { + be.m.Lock() + defer be.m.Unlock() + + if _, ok := be.data[entry{t, name}]; !ok { + return nil, errors.New("no such data") + } + + return readCloser{bytes.NewReader(be.data[entry{t, name}])}, nil +} + +func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) (io.ReadCloser, error) { + be.m.Lock() + defer be.m.Unlock() + + if _, ok := be.data[entry{t, name}]; !ok { + return nil, errors.New("no such data") + } + + buf := be.data[entry{t, name}] + if offset > uint(len(buf)) { + return nil, errors.New("offset beyond end of file") + } + + buf = buf[offset:] + + if length > uint(len(buf)) { + length = uint(len(buf)) + } + + buf = buf[:length] + + return readCloser{bytes.NewReader(buf)}, nil +} + +func memRemove(be *MemoryBackend, t Type, name string) error { + be.m.Lock() + defer be.m.Unlock() + + if _, ok := be.data[entry{t, name}]; !ok { + return errors.New("no such data") + } + + delete(be.data, entry{t, name}) + + return nil +} + +func memList(be *MemoryBackend, t Type, done <-chan struct{}) <-chan string { + be.m.Lock() + defer be.m.Unlock() + + ch := make(chan string) + + var ids []string + for entry := range be.data { + if entry.Type != t { + continue + } + ids = append(ids, entry.Name) + } + + sort.Strings(ids) + + go func() { + defer close(ch) + for _, id := range ids { + select { + case ch <- id: + case <-done: + return + } + } + }() + + return ch +} diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go new file mode 100644 index 000000000..c5b43415c --- /dev/null +++ b/backend/mem_backend_test.go @@ -0,0 +1,12 @@ +package backend_test + +import ( + "testing" + + "github.com/restic/restic/backend" +) + +func TestMemoryBackend(t *testing.T) { + be := backend.NewMemoryBackend() + testBackend(be, t) +} From 538e5878a1fee30a24fe2e1586af3aca4d98c813 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 16:30:13 +0100 Subject: [PATCH 08/13] add debug logging to MemoryBackend --- backend/mem_backend.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/backend/mem_backend.go b/backend/mem_backend.go index 213ddb68f..e6b51cf8c 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -6,6 +6,8 @@ import ( "io" "sort" "sync" + + "github.com/restic/restic/debug" ) type entry struct { @@ -55,6 +57,8 @@ func NewMemoryBackend() *MemoryBackend { return memList(be, t, done) } + debug.Log("MemoryBackend.New", "created new memory backend") + return be } @@ -74,6 +78,8 @@ func memTest(be *MemoryBackend, t Type, name string) (bool, error) { be.m.Lock() defer be.m.Unlock() + debug.Log("MemoryBackend.Test", "test %v %v", t, name) + if _, ok := be.data[entry{t, name}]; ok { return true, nil } @@ -117,6 +123,8 @@ func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() + debug.Log("MemoryBackend.Get", "get %v %v", t, name) + if _, ok := be.data[entry{t, name}]; !ok { return nil, errors.New("no such data") } @@ -128,6 +136,8 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( be.m.Lock() defer be.m.Unlock() + debug.Log("MemoryBackend.GetReader", "get %v %v", t, name) + if _, ok := be.data[entry{t, name}]; !ok { return nil, errors.New("no such data") } @@ -152,6 +162,8 @@ func memRemove(be *MemoryBackend, t Type, name string) error { be.m.Lock() defer be.m.Unlock() + debug.Log("MemoryBackend.Remove", "get %v %v", t, name) + if _, ok := be.data[entry{t, name}]; !ok { return errors.New("no such data") } @@ -177,6 +189,8 @@ func memList(be *MemoryBackend, t Type, done <-chan struct{}) <-chan string { sort.Strings(ids) + debug.Log("MemoryBackend.List", "list %v: %v", t, ids) + go func() { defer close(ch) for _, id := range ids { From 480054bc3ad4c35aa7c440f7f8980f6be0a7042e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 16:38:58 +0100 Subject: [PATCH 09/13] MemoryBackend: handle config correctly, add tests for that --- backend/backend_test.go | 27 +++++++++++++++++++++++++++ backend/mem_backend.go | 17 ++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/backend_test.go b/backend/backend_test.go index ef928cf8e..f38fbced5 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -12,7 +12,34 @@ import ( . "github.com/restic/restic/test" ) +func testBackendConfig(b backend.Backend, t *testing.T) { + // create config and read it back + _, err := b.Get(backend.Config, "") + Assert(t, err != nil, "did not get expected error for non-existing config") + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte("Config")) + OK(t, err) + OK(t, blob.Finalize(backend.Config, "")) + + // try accessing the config with different names, should all return the + // same config + for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { + rd, err := b.Get(backend.Config, name) + Assert(t, err == nil, "unable to read config") + + buf, err := ioutil.ReadAll(rd) + OK(t, err) + OK(t, rd.Close()) + Assert(t, string(buf) == "Config", "wrong data returned for config") + } +} + func testBackend(b backend.Backend, t *testing.T) { + testBackendConfig(b, t) + for _, tpe := range []backend.Type{ backend.Data, backend.Key, backend.Lock, backend.Snapshot, backend.Index, diff --git a/backend/mem_backend.go b/backend/mem_backend.go index e6b51cf8c..61f23e4ca 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -103,11 +103,18 @@ func (e *tempMemEntry) Size() uint { } func (e *tempMemEntry) Finalize(t Type, name string) error { + if t == Config { + name = "" + } + + debug.Log("MemoryBackend", "save blob %p as %v %v", e, t, name) return e.be.insert(t, name, e.data.Bytes()) } func memCreate(be *MemoryBackend) (Blob, error) { - return &tempMemEntry{be: be}, nil + blob := &tempMemEntry{be: be} + debug.Log("MemoryBackend.Create", "create new blob %p", blob) + return blob, nil } // readCloser wraps a reader and adds a noop Close method. @@ -123,6 +130,10 @@ func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() + if t == Config { + name = "" + } + debug.Log("MemoryBackend.Get", "get %v %v", t, name) if _, ok := be.data[entry{t, name}]; !ok { @@ -136,6 +147,10 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( be.m.Lock() defer be.m.Unlock() + if t == Config { + name = "" + } + debug.Log("MemoryBackend.GetReader", "get %v %v", t, name) if _, ok := be.data[entry{t, name}]; !ok { From 26697a02234b0c1a2d9af56e6dc22e263f921564 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 17:27:02 +0100 Subject: [PATCH 10/13] Fix MemoryBackend GetReader() method --- backend/mem_backend.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/mem_backend.go b/backend/mem_backend.go index 61f23e4ca..35e5df643 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -151,7 +151,7 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( name = "" } - debug.Log("MemoryBackend.GetReader", "get %v %v", t, name) + debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length) if _, ok := be.data[entry{t, name}]; !ok { return nil, errors.New("no such data") @@ -164,11 +164,13 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( buf = buf[offset:] - if length > uint(len(buf)) { - length = uint(len(buf)) - } + if length > 0 { + if length > uint(len(buf)) { + length = uint(len(buf)) + } - buf = buf[:length] + buf = buf[:length] + } return readCloser{bytes.NewReader(buf)}, nil } From 4f6bc754b829b2eb71dcf63ff7dd99766bfaee4a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 20:45:03 +0100 Subject: [PATCH 11/13] MemBackend: Add Delete() and more debug --- backend/mem_backend.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/mem_backend.go b/backend/mem_backend.go index 35e5df643..9ba17a72f 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -57,6 +57,14 @@ func NewMemoryBackend() *MemoryBackend { return memList(be, t, done) } + be.MockBackend.DeleteFn = func() error { + be.m.Lock() + defer be.m.Unlock() + + be.data = make(memMap) + return nil + } + debug.Log("MemoryBackend.New", "created new memory backend") return be @@ -107,7 +115,7 @@ func (e *tempMemEntry) Finalize(t Type, name string) error { name = "" } - debug.Log("MemoryBackend", "save blob %p as %v %v", e, t, name) + debug.Log("MemoryBackend", "save blob %p (%d bytes) as %v %v", e, len(e.data.Bytes()), t, name) return e.be.insert(t, name, e.data.Bytes()) } From b841eb4c5458ad85da66d9040c5b4e69f54fb9df Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 20:42:20 +0100 Subject: [PATCH 12/13] crypto: check key for validity --- crypto/crypto.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crypto/crypto.go b/crypto/crypto.go index 4a1a66bed..559569679 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -233,6 +233,10 @@ var ErrInvalidCiphertext = errors.New("invalid ciphertext, same slice used for p // necessary. ciphertext and plaintext may not point to (exactly) the same // slice or non-intersecting slices. func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) { + if !ks.Valid() { + return nil, errors.New("invalid key") + } + ciphertext = ciphertext[:cap(ciphertext)] // test for same slice, if possible @@ -271,6 +275,10 @@ func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) { // IV || Ciphertext || MAC. plaintext and ciphertext may point to (exactly) the // same slice. func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error) { + if !ks.Valid() { + return nil, errors.New("invalid key") + } + // check for plausible length if len(ciphertextWithMac) < ivSize+macSize { panic("trying to decrypt invalid data: ciphertext too small") From 141d400b4ac90c8e6ad136cfd23704bd98291c24 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 22 Nov 2015 22:08:23 +0100 Subject: [PATCH 13/13] test: improvements --- test/helpers.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/test/helpers.go b/test/helpers.go index eab73ddfb..7abb536b3 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -4,10 +4,10 @@ import ( "bytes" "compress/bzip2" "compress/gzip" + "crypto/rand" "fmt" "io" "io/ioutil" - "math/rand" "os" "os/exec" "path/filepath" @@ -15,6 +15,8 @@ import ( "runtime" "testing" + mrand "math/rand" + "github.com/restic/restic/backend" "github.com/restic/restic/backend/local" "github.com/restic/restic/repository" @@ -76,7 +78,7 @@ func ParseID(s string) backend.ID { func Random(seed, count int) []byte { buf := make([]byte, count) - rnd := rand.New(rand.NewSource(int64(seed))) + rnd := mrand.New(mrand.NewSource(int64(seed))) for i := 0; i < count; i++ { buf[i] = byte(rnd.Uint32()) } @@ -90,6 +92,14 @@ func RandomReader(seed, size int) *bytes.Reader { return bytes.NewReader(Random(seed, size)) } +// GenRandom returns a []byte filled with up to 1000 random bytes. +func GenRandom(t testing.TB) []byte { + buf := make([]byte, mrand.Intn(1000)) + _, err := io.ReadFull(rand.Reader, buf) + OK(t, err) + return buf +} + // SetupTarTestFixture extracts the tarFile to outputDir. func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) { input, err := os.Open(tarFile)