From b335de6b8143c3dbe7df0dfe694d1524510c0134 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 15:23:28 +0200 Subject: [PATCH 01/13] Add comments to exported methods --- restorer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/restorer.go b/restorer.go index 740677473..cebd9c894 100644 --- a/restorer.go +++ b/restorer.go @@ -12,6 +12,7 @@ import ( "github.com/juju/errors" ) +// Restorer is used to restore a snapshot to a directory. type Restorer struct { s *server.Server sn *Snapshot @@ -103,6 +104,7 @@ func (res *Restorer) RestoreTo(dir string) error { return res.restoreTo(dir, "", res.sn.Tree) } +// Snapshot returns the snapshot this restorer is configured to use. func (res *Restorer) Snapshot() *Snapshot { return res.sn } From 819b6da7620b52774dead3d96e6407f9d29bbeb0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 15:31:31 +0200 Subject: [PATCH 02/13] restorer: unexport archive pipe --- archiver.go | 6 +++--- archiver_int_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/archiver.go b/archiver.go index 8632b82fd..c3517c0b1 100644 --- a/archiver.go +++ b/archiver.go @@ -459,7 +459,7 @@ func (arch *Archiver) dirWorker(wg *sync.WaitGroup, p *Progress, done <-chan str } } -type ArchivePipe struct { +type archivePipe struct { Old <-chan WalkTreeJob New <-chan pipe.Job } @@ -499,7 +499,7 @@ type archiveJob struct { new pipe.Job } -func (a *ArchivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) { +func (a *archivePipe) compare(done <-chan struct{}, out chan<- pipe.Job) { defer func() { close(out) debug.Log("ArchivePipe.compare", "done") @@ -638,7 +638,7 @@ func (arch *Archiver) Snapshot(p *Progress, paths []string, pid backend.ID) (*Sn return nil, nil, err } - jobs := ArchivePipe{} + jobs := archivePipe{} // use parent snapshot (if some was given) if pid != nil { diff --git a/archiver_int_test.go b/archiver_int_test.go index 948d44ba6..d26a9ce53 100644 --- a/archiver_int_test.go +++ b/archiver_int_test.go @@ -115,7 +115,7 @@ func TestArchivePipe(t *testing.T) { go testTreeWalker(done, treeCh) go testPipeWalker(done, pipeCh) - p := ArchivePipe{Old: treeCh, New: pipeCh} + p := archivePipe{Old: treeCh, New: pipeCh} ch := make(chan pipe.Job) From 0a65901e185a38c539aec8bd932a07aba683e146 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 15:49:14 +0200 Subject: [PATCH 03/13] cache: unexport internal functions --- cache.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cache.go b/cache.go index 53e14873e..7d55337be 100644 --- a/cache.go +++ b/cache.go @@ -99,7 +99,7 @@ func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error { } func (c *Cache) Clear(s *server.Server) error { - list, err := c.List(backend.Snapshot) + list, err := c.list(backend.Snapshot) if err != nil { return err } @@ -120,19 +120,19 @@ func (c *Cache) Clear(s *server.Server) error { return nil } -type CacheEntry struct { +type cacheEntry struct { ID backend.ID Subtype string } -func (c CacheEntry) String() string { +func (c cacheEntry) String() string { if c.Subtype != "" { return c.ID.Str() + "." + c.Subtype } return c.ID.Str() } -func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { +func (c *Cache) list(t backend.Type) ([]cacheEntry, error) { var dir string switch t { @@ -145,7 +145,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { fd, err := os.Open(dir) if err != nil { if os.IsNotExist(err) { - return []CacheEntry{}, nil + return []cacheEntry{}, nil } return nil, err } @@ -156,7 +156,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { return nil, err } - entries := make([]CacheEntry, 0, len(fis)) + entries := make([]cacheEntry, 0, len(fis)) for _, fi := range fis { parts := strings.SplitN(fi.Name(), ".", 2) @@ -168,7 +168,7 @@ func (c *Cache) List(t backend.Type) ([]CacheEntry, error) { continue } - e := CacheEntry{ID: id} + e := cacheEntry{ID: id} if len(parts) == 2 { e.Subtype = parts[1] From ff4dd1f1fb1100d231129acd0ea215fe5a4d1812 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 15:51:40 +0200 Subject: [PATCH 04/13] Add more comments --- archiver.go | 16 ++++++++++++---- cache.go | 5 +++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/archiver.go b/archiver.go index c3517c0b1..301ec9471 100644 --- a/archiver.go +++ b/archiver.go @@ -28,6 +28,7 @@ const ( var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err } var archiverAllowAllFiles = func(string, os.FileInfo) bool { return true } +// Archiver is used to backup a set of directories. type Archiver struct { s *server.Server @@ -37,6 +38,7 @@ type Archiver struct { Filter func(item string, fi os.FileInfo) bool } +// NewArchiver returns a new archiver. func NewArchiver(s *server.Server) *Archiver { arch := &Archiver{ s: s, @@ -53,6 +55,7 @@ func NewArchiver(s *server.Server) *Archiver { return arch } +// Save stores a blob read from rd in the server. func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error { debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str()) @@ -73,6 +76,7 @@ func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Re return nil } +// SaveTreeJSON stores a tree in the server. func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) { data, err := json.Marshal(item) if err != nil { @@ -619,7 +623,9 @@ func (j archiveJob) Copy() pipe.Job { return j.new } -func (arch *Archiver) Snapshot(p *Progress, paths []string, pid backend.ID) (*Snapshot, backend.ID, error) { +// Snapshot creates a snapshot of the given paths. If parentID is set, this is +// used to compare the files with. +func (arch *Archiver) Snapshot(p *Progress, paths []string, parentID backend.ID) (*Snapshot, backend.ID, error) { debug.Log("Archiver.Snapshot", "start for %v", paths) debug.Break("Archiver.Snapshot") @@ -641,11 +647,11 @@ func (arch *Archiver) Snapshot(p *Progress, paths []string, pid backend.ID) (*Sn jobs := archivePipe{} // use parent snapshot (if some was given) - if pid != nil { - sn.Parent = pid + if parentID != nil { + sn.Parent = parentID // load parent snapshot - parent, err := LoadSnapshot(arch.s, pid) + parent, err := LoadSnapshot(arch.s, parentID) if err != nil { return nil, nil, err } @@ -745,6 +751,8 @@ func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } +// Scan traverses the dirs to collect Stat information while emitting progress +// information with p. func Scan(dirs []string, p *Progress) (Stat, error) { p.Start() defer p.Done() diff --git a/cache.go b/cache.go index 7d55337be..fb02024b3 100644 --- a/cache.go +++ b/cache.go @@ -13,6 +13,7 @@ import ( "github.com/restic/restic/server" ) +// Cache is used to handle the local cache. type Cache struct { base string } @@ -29,6 +30,7 @@ func NewCache(be backend.Identifier) (*Cache, error) { return &Cache{base: basedir}, nil } +// Has checks if the local cache has the id. func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -51,6 +53,7 @@ func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) return true, nil } +// Store returns an io.WriteCloser that is used to save new information to. func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -73,6 +76,7 @@ func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCl return file, nil } +// Load returns information from the cache. func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -98,6 +102,7 @@ func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error { return err } +// Clear removes information from the cache that isn't present in the server any more. func (c *Cache) Clear(s *server.Server) error { list, err := c.list(backend.Snapshot) if err != nil { From 0ca678228d4d96963ff696ce753b47db29a5fe35 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:12:19 +0200 Subject: [PATCH 05/13] node: add comments --- node.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/node.go b/node.go index 39f1da2cb..958fc16e4 100644 --- a/node.go +++ b/node.go @@ -17,6 +17,7 @@ import ( "github.com/restic/restic/server" ) +// Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` Type string `json:"type"` @@ -62,6 +63,7 @@ func (node Node) Tree() *Tree { return node.tree } +// NodeFromFileInfo returns a new node from the given path and FileInfo. func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { node := &Node{ path: path, @@ -100,6 +102,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string { return "" } +// CreatAt creates the node at the given path and restores all the meta data. func (node *Node) CreateAt(path string, s *server.Server) error { switch node.Type { case "dir": From 1f94ac724fe01365b23ae114222dfa6c42fdf3e1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:13:56 +0200 Subject: [PATCH 06/13] backend: rename LimitReader to LimitReadCloser --- backend/generic.go | 10 ++++++---- backend/local/local.go | 2 +- backend/sftp/sftp.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index 0244cdab1..c7d569972 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -100,7 +100,7 @@ outer: // wrap around io.LimitedReader that implements io.ReadCloser type blobReader struct { - f io.Closer + cl io.Closer rd io.Reader closed bool } @@ -120,7 +120,7 @@ func (l *blobReader) Close() error { } if !l.closed { - err := l.f.Close() + err := l.cl.Close() l.closed = true return err } @@ -128,6 +128,8 @@ func (l *blobReader) Close() error { return nil } -func LimitReader(f io.ReadCloser, n int64) *blobReader { - return &blobReader{f: f, rd: io.LimitReader(f, n)} +// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also +// implements the Close() method. +func LimitReadCloser(r io.ReadCloser, n int64) *blobReader { + return &blobReader{cl: r, rd: io.LimitReader(r, n)} } diff --git a/backend/local/local.go b/backend/local/local.go index 0fbd5fcf7..a50d2ef31 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -312,7 +312,7 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. return f, nil } - return backend.LimitReader(f, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } // Test returns true if a blob of the given type and name exists in the backend. diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 09dad6242..5c850ce77 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -446,7 +446,7 @@ func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.R return f, nil } - return backend.LimitReader(f, int64(length)), nil + return backend.LimitReadCloser(f, int64(length)), nil } // Test returns true if a blob of the given type and name exists in the backend. From cf62eb3e15f34899771ca8867ea5f32b673f6711 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:19:16 +0200 Subject: [PATCH 07/13] chunker: unexport WindowSize and AverageBits --- chunker/chunker.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/chunker/chunker.go b/chunker/chunker.go index eb92a50c7..a319c71f8 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -12,16 +12,16 @@ const ( MiB = 1024 * KiB // WindowSize is the size of the sliding window. - WindowSize = 64 + windowSize = 64 // aim to create chunks of 20 bits or about 1MiB on average. - AverageBits = 20 + averageBits = 20 // Chunks should be in the range of 512KiB to 8MiB. MinSize = 512 * KiB MaxSize = 8 * MiB - splitmask = (1 << AverageBits) - 1 + splitmask = (1 << averageBits) - 1 ) type tables struct { @@ -61,7 +61,7 @@ type Chunker struct { rd io.Reader closed bool - window [WindowSize]byte + window [windowSize]byte wpos int buf []byte @@ -97,7 +97,7 @@ func (c *Chunker) Reset(rd io.Reader, p Pol) { c.fillTables() c.rd = rd - for i := 0; i < WindowSize; i++ { + for i := 0; i < windowSize; i++ { c.window[i] = 0 } c.closed = false @@ -116,7 +116,7 @@ func (c *Chunker) Reset(rd io.Reader, p Pol) { } // do not start a new chunk unless at least MinSize bytes have been read - c.pre = MinSize - WindowSize + c.pre = MinSize - windowSize } // Calculate out_table and mod_table for optimization. Must be called only @@ -154,7 +154,7 @@ func (c *Chunker) fillTables() { var h Pol h = appendByte(h, byte(b), c.pol) - for i := 0; i < WindowSize-1; i++ { + for i := 0; i < windowSize-1; i++ { h = appendByte(h, 0, c.pol) } c.tables.out[b] = h @@ -246,7 +246,7 @@ func (c *Chunker) Next() (*Chunk, error) { out := c.window[c.wpos] c.window[c.wpos] = b c.digest ^= uint64(c.tables.out[out]) - c.wpos = (c.wpos + 1) % WindowSize + c.wpos = (c.wpos + 1) % windowSize // c.append(b) index := c.digest >> c.polShift @@ -284,7 +284,7 @@ func (c *Chunker) Next() (*Chunk, error) { c.Reset(c.rd, c.pol) c.pos = pos c.start = pos - c.pre = MinSize - WindowSize + c.pre = MinSize - windowSize return chunk, nil } @@ -330,7 +330,7 @@ func (c *Chunker) slide(b byte) { out := c.window[c.wpos] c.window[c.wpos] = b c.digest ^= uint64(c.tables.out[out]) - c.wpos = (c.wpos + 1) % WindowSize + c.wpos = (c.wpos + 1) % windowSize c.append(b) } From 6fe38b0cd8ba3767d381f2367daa0061e166f2c8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:22:43 +0200 Subject: [PATCH 08/13] fix comments to have the standard form --- backend/id.go | 2 +- chunker/chunker.go | 7 ++++--- node.go | 2 +- server/key.go | 4 ++-- server/server.go | 2 -- test/helpers.go | 4 ++-- 6 files changed, 10 insertions(+), 11 deletions(-) diff --git a/backend/id.go b/backend/id.go index 261f4802d..e960b8eaf 100644 --- a/backend/id.go +++ b/backend/id.go @@ -10,7 +10,7 @@ import ( // IDSize contains the size of an ID, in bytes. const IDSize = hashSize -// References content within a repository. +// ID references content within a repository. type ID []byte // ParseID converts the given string to an ID. diff --git a/chunker/chunker.go b/chunker/chunker.go index a319c71f8..9d19a2e2a 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -17,8 +17,9 @@ const ( // aim to create chunks of 20 bits or about 1MiB on average. averageBits = 20 - // Chunks should be in the range of 512KiB to 8MiB. + // MinSize is the minimal size of a chunk. MinSize = 512 * KiB + // MaxSize is the maximal size of a chunk. MaxSize = 8 * MiB splitmask = (1 << averageBits) - 1 @@ -39,7 +40,7 @@ func init() { cache.entries = make(map[Pol]*tables) } -// A chunk is one content-dependent chunk of bytes whose end was cut when the +// Chunk is one content-dependent chunk of bytes whose end was cut when the // Rabin Fingerprint had the value stored in Cut. type Chunk struct { Start uint @@ -52,7 +53,7 @@ func (c Chunk) Reader(r io.ReaderAt) io.Reader { return io.NewSectionReader(r, int64(c.Start), int64(c.Length)) } -// A chunker internally holds everything needed to split content. +// Chunker split content with Rabin Fingerprints. type Chunker struct { pol Pol polShift uint diff --git a/node.go b/node.go index 958fc16e4..3060d54fb 100644 --- a/node.go +++ b/node.go @@ -102,7 +102,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string { return "" } -// CreatAt creates the node at the given path and restores all the meta data. +// CreateAt creates the node at the given path and restores all the meta data. func (node *Node) CreateAt(path string, s *server.Server) error { switch node.Type { case "dir": diff --git a/server/key.go b/server/key.go index bcecb7719..68f00c481 100644 --- a/server/key.go +++ b/server/key.go @@ -258,13 +258,13 @@ func (k *Key) DecryptFrom(rd io.Reader) (io.ReadCloser, error) { return crypto.DecryptFrom(k.master, rd) } -// Master() returns the master keys for this repository. Only included for +// Master returns the master keys for this repository. Only included for // debug purposes. func (k *Key) Master() *crypto.Key { return k.master } -// User() returns the user keys for this key. Only included for debug purposes. +// User returns the user keys for this key. Only included for debug purposes. func (k *Key) User() *crypto.Key { return k.user } diff --git a/server/server.go b/server/server.go index da1974a99..e2a84650c 100644 --- a/server/server.go +++ b/server/server.go @@ -422,12 +422,10 @@ func (s *Server) Flush() error { return nil } -// Returns the backend used for this server. func (s *Server) Backend() backend.Backend { return s.be } -// Returns the index of this server. func (s *Server) Index() *Index { return s.idx } diff --git a/test/helpers.go b/test/helpers.go index a3e5a743c..b99f56dec 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -21,7 +21,7 @@ func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { } } -// ok fails the test if an err is not nil. +// OK fails the test if an err is not nil. func OK(tb testing.TB, err error) { if err != nil { _, file, line, _ := runtime.Caller(1) @@ -30,7 +30,7 @@ func OK(tb testing.TB, err error) { } } -// equals fails the test if exp is not equal to act. +// Equals fails the test if exp is not equal to act. func Equals(tb testing.TB, exp, act interface{}) { if !reflect.DeepEqual(exp, act) { _, file, line, _ := runtime.Caller(1) From cd0c537e7eaa49fb6168b1888df2e854f7ca43a0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:25:26 +0200 Subject: [PATCH 09/13] remove unused file --- zerrors_linux.go | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 zerrors_linux.go diff --git a/zerrors_linux.go b/zerrors_linux.go deleted file mode 100644 index c90a86293..000000000 --- a/zerrors_linux.go +++ /dev/null @@ -1,5 +0,0 @@ -package restic - -// Add constant O_PATH missing from Go1.3, will be added to Go1.4 according to -// https://code.google.com/p/go/issues/detail?id=7830 -const O_PATH = 010000000 From 71f3fb2dac947ec00d358e347a1532291596935b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:36:48 +0200 Subject: [PATCH 10/13] Rename variables in copyJobs() --- archiver.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/archiver.go b/archiver.go index 301ec9471..7f2cc11a8 100644 --- a/archiver.go +++ b/archiver.go @@ -469,30 +469,30 @@ type archivePipe struct { } func copyJobs(done <-chan struct{}, in <-chan pipe.Job, out chan<- pipe.Job) { - i := in - o := out - - o = nil - var ( - j pipe.Job - ok bool + // disable sending on the outCh until we received a job + outCh chan<- pipe.Job + // enable receiving from in + inCh = in + job pipe.Job + ok bool ) + for { select { case <-done: return - case j, ok = <-i: + case job, ok = <-inCh: if !ok { - // in ch closed, we're done - debug.Log("copyJobs", "in channel closed, we're done") + // input channel closed, we're done + debug.Log("copyJobs", "input channel closed, we're done") return } - i = nil - o = out - case o <- j: - o = nil - i = in + inCh = nil + outCh = out + case outCh <- job: + outCh = nil + inCh = in } } } From d66780a45d5989b1b1e4bb41e650953164479b53 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:39:24 +0200 Subject: [PATCH 11/13] Remove unused global variable --- archiver.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/archiver.go b/archiver.go index 7f2cc11a8..e38e2df94 100644 --- a/archiver.go +++ b/archiver.go @@ -20,9 +20,8 @@ import ( ) const ( - maxConcurrentBlobs = 32 - maxConcurrency = 10 - maxConcurrencyPreload = 20 + maxConcurrentBlobs = 32 + maxConcurrency = 10 ) var archiverAbortOnAllErrors = func(str string, fi os.FileInfo, err error) error { return err } From c9664e9eacb8c2c3821fadf1a2e1af02e63e92cb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 16:55:29 +0200 Subject: [PATCH 12/13] Remove unused function saveTree() --- archiver.go | 93 ----------------------------------------------------- 1 file changed, 93 deletions(-) diff --git a/archiver.go b/archiver.go index e38e2df94..31d4966aa 100644 --- a/archiver.go +++ b/archiver.go @@ -212,99 +212,6 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error { return err } -func (arch *Archiver) saveTree(p *Progress, t *Tree) (backend.ID, error) { - debug.Log("Archiver.saveTree", "saveTree(%v)\n", t) - var wg sync.WaitGroup - - // TODO: do all this in parallel - for _, node := range t.Nodes { - if node.tree != nil { - id, err := arch.saveTree(p, node.tree) - if err != nil { - return nil, err - } - node.Subtree = id - p.Report(Stat{Dirs: 1}) - } else if node.Type == "file" { - if len(node.Content) > 0 { - removeContent := false - - // check content - for _, id := range node.Content { - packID, _, _, _, err := arch.s.Index().Lookup(id) - if err != nil { - debug.Log("Archiver.saveTree", "unable to find storage id for data blob %v: %v", id.Str(), err) - arch.Error(node.path, nil, fmt.Errorf("unable to find storage id for data blob %v: %v", id.Str(), err)) - removeContent = true - continue - } - - if ok, err := arch.s.Test(backend.Data, packID.String()); !ok || err != nil { - debug.Log("Archiver.saveTree", "pack %v of blob %v not in repository (error is %v)", packID, id, err) - arch.Error(node.path, nil, fmt.Errorf("pack %v of blob %v not in repository (error is %v)", packID, id, err)) - removeContent = true - } - } - - if removeContent { - debug.Log("Archiver.saveTree", "removing content for %s", node.path) - node.Content = node.Content[:0] - } - } - - if len(node.Content) == 0 { - // start goroutine - wg.Add(1) - go func(n *Node) { - defer wg.Done() - - n.err = arch.SaveFile(p, n) - p.Report(Stat{Files: 1}) - }(node) - } - } - } - - wg.Wait() - - usedIDs := backend.NewIDSet() - - // check for invalid file nodes - for _, node := range t.Nodes { - if node.Type == "file" && node.Content == nil && node.err == nil { - return nil, fmt.Errorf("node %v has empty content", node.Name) - } - - // remember used hashes - if node.Type == "file" && node.Content != nil { - for _, id := range node.Content { - usedIDs.Insert(id) - } - } - - if node.Type == "dir" && node.Subtree != nil { - usedIDs.Insert(node.Subtree) - } - - if node.err != nil { - err := arch.Error(node.path, nil, node.err) - if err != nil { - return nil, err - } - - // save error message in node - node.Error = node.err.Error() - } - } - - id, err := arch.SaveTreeJSON(t) - if err != nil { - return nil, err - } - - return id, nil -} - func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan struct{}, entCh <-chan pipe.Entry) { defer func() { debug.Log("Archiver.fileWorker", "done") From ba1125df1075949636a347c00fc5c24c49791171 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 2 May 2015 17:01:31 +0200 Subject: [PATCH 13/13] Cleanup commets with feedback from @fw42 --- archiver.go | 3 ++- cache.go | 9 ++++++--- chunker/chunker.go | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/archiver.go b/archiver.go index 31d4966aa..ca8542b65 100644 --- a/archiver.go +++ b/archiver.go @@ -530,7 +530,8 @@ func (j archiveJob) Copy() pipe.Job { } // Snapshot creates a snapshot of the given paths. If parentID is set, this is -// used to compare the files with. +// used to compare the files to the ones archived at the time this snapshot was +// taken. func (arch *Archiver) Snapshot(p *Progress, paths []string, parentID backend.ID) (*Snapshot, backend.ID, error) { debug.Log("Archiver.Snapshot", "start for %v", paths) diff --git a/cache.go b/cache.go index fb02024b3..24f7a17fd 100644 --- a/cache.go +++ b/cache.go @@ -13,7 +13,7 @@ import ( "github.com/restic/restic/server" ) -// Cache is used to handle the local cache. +// Cache is used to locally cache items from a server. type Cache struct { base string } @@ -53,7 +53,9 @@ func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error) return true, nil } -// Store returns an io.WriteCloser that is used to save new information to. +// Store returns an io.WriteCloser that is used to save new information to the +// cache. The returned io.WriteCloser must be closed by the caller after all +// data has been written. func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { @@ -76,7 +78,8 @@ func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCl return file, nil } -// Load returns information from the cache. +// Load returns information from the cache. The returned io.ReadCloser must be +// closed by the caller. func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadCloser, error) { filename, err := c.filename(t, subtype, id) if err != nil { diff --git a/chunker/chunker.go b/chunker/chunker.go index 9d19a2e2a..9e016e4dd 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -53,7 +53,7 @@ func (c Chunk) Reader(r io.ReaderAt) io.Reader { return io.NewSectionReader(r, int64(c.Start), int64(c.Length)) } -// Chunker split content with Rabin Fingerprints. +// Chunker splits content with Rabin Fingerprints. type Chunker struct { pol Pol polShift uint