diff --git a/archiver.go b/archiver.go index b242d82fb..285ba25c2 100644 --- a/archiver.go +++ b/archiver.go @@ -61,6 +61,11 @@ func NewArchiver(s Server) (*Archiver, error) { // Preload loads all tree objects from repository and adds all blobs that are // still available to the map for deduplication. func (arch *Archiver) Preload(p *Progress) error { + cache, err := NewCache() + if err != nil { + return err + } + p.Start() defer p.Done() @@ -69,10 +74,28 @@ func (arch *Archiver) Preload(p *Progress) error { // load all trees, in parallel worker := func(wg *sync.WaitGroup, c <-chan backend.ID) { for id := range c { - tree, err := LoadTree(arch.s, id) - // ignore error and advance to next tree - if err != nil { - return + var tree *Tree + + // load from cache + var t Tree + rd, err := cache.Load(backend.Tree, id) + if err == nil { + debug.Log("Archiver.Preload", "tree %v cached", id.Str()) + tree = &t + dec := json.NewDecoder(rd) + err = dec.Decode(&t) + + if err != nil { + continue + } + } else { + debug.Log("Archiver.Preload", "tree %v not cached: %v", id.Str(), err) + + tree, err = LoadTree(arch.s, id) + // ignore error and advance to next tree + if err != nil { + continue + } } debug.Log("Archiver.Preload", "load tree %v with %d blobs", id, tree.Map.Len()) @@ -93,7 +116,7 @@ func (arch *Archiver) Preload(p *Progress) error { // list ids trees := 0 - err := arch.s.EachID(backend.Tree, func(id backend.ID) { + err = arch.s.EachID(backend.Tree, func(id backend.ID) { trees++ if trees%1000 == 0 { diff --git a/archiver_test.go b/archiver_test.go index bbc9be180..81f2b80df 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -202,10 +202,6 @@ func TestArchivePreload(t *testing.T) { archiveWithPreload(t) } -func BenchmarkArchivePreload(b *testing.B) { - archiveWithPreload(b) -} - func BenchmarkPreload(t *testing.B) { if *benchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverPreload") @@ -233,3 +229,33 @@ func BenchmarkPreload(t *testing.B) { ok(t, arch2.Preload(nil)) } } + +func BenchmarkLoadTree(t *testing.B) { + if *benchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping TestArchiverPreload") + } + + be := setupBackend(t) + defer teardownBackend(t, be) + key := setupKey(t, be, "geheim") + server := restic.NewServerWithKey(be, key) + + // archive a few files + arch, err := restic.NewArchiver(server) + ok(t, err) + sn, _, err := arch.Snapshot(nil, *benchArchiveDirectory, nil) + ok(t, err) + t.Logf("archived snapshot %v", sn.ID()) + + // start benchmark + t.ResetTimer() + + list, err := server.List(backend.Tree) + ok(t, err) + list = list[:10] + + for i := 0; i < t.N; i++ { + _, err := restic.LoadTree(server, list[0]) + ok(t, err) + } +} diff --git a/cache.go b/cache.go new file mode 100644 index 000000000..9ecad0339 --- /dev/null +++ b/cache.go @@ -0,0 +1,92 @@ +package restic + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "github.com/restic/restic/backend" +) + +type Cache struct { + base string +} + +func NewCache() (*Cache, error) { + dir, err := GetCacheDir() + if err != nil { + return nil, err + } + + return &Cache{base: dir}, nil +} + +func (c *Cache) Has(t backend.Type, id backend.ID) (bool, error) { + // try to open file + filename, err := c.filename(t, id) + if err != nil { + return false, err + } + + fd, err := os.Open(filename) + defer fd.Close() + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (c *Cache) Store(t backend.Type, id backend.ID, rd io.Reader) error { + filename, err := c.filename(t, id) + if err != nil { + return err + } + + dirname := filepath.Dir(filename) + err = os.MkdirAll(dirname, 0700) + if err != nil { + return err + } + + file, err := os.Create(filename) + defer file.Close() + if err != nil { + return err + } + + _, err = io.Copy(file, rd) + return err +} + +func (c *Cache) Load(t backend.Type, id backend.ID) (io.ReadCloser, error) { + // try to open file + filename, err := c.filename(t, id) + if err != nil { + return nil, err + } + + return os.Open(filename) +} + +// Construct file name for given Type. +func (c *Cache) filename(t backend.Type, id backend.ID) (string, error) { + cachedir, err := GetCacheDir() + if err != nil { + return "", err + } + + switch t { + case backend.Snapshot: + return filepath.Join(cachedir, "snapshots", id.String()), nil + case backend.Tree: + return filepath.Join(cachedir, "trees", id.String()), nil + } + + return "", fmt.Errorf("cache not supported for type %v", t) +} diff --git a/cache_linux.go b/cache_linux.go new file mode 100644 index 000000000..816fa2416 --- /dev/null +++ b/cache_linux.go @@ -0,0 +1,49 @@ +package restic + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/restic/restic/debug" +) + +// GetCacheDir returns the cache directory according to XDG basedir spec, see +// http://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html +func GetCacheDir() (string, error) { + xdgcache := os.Getenv("XDG_CACHE_HOME") + home := os.Getenv("HOME") + + if xdgcache == "" && home == "" { + return "", errors.New("unable to locate cache directory (XDG_CACHE_HOME and HOME unset)") + } + + cachedir := "" + if xdgcache != "" { + cachedir = filepath.Join(xdgcache, "restic") + } else if home != "" { + cachedir = filepath.Join(home, ".cache", "restic") + } + + fi, err := os.Stat(cachedir) + if os.IsNotExist(err) { + err = os.MkdirAll(cachedir, 0700) + if err != nil { + return "", err + } + + fi, err = os.Stat(cachedir) + debug.Log("getCacheDir", "create cache dir %v", cachedir) + } + + if err != nil { + return "", err + } + + if !fi.IsDir() { + return "", fmt.Errorf("cache dir %v is not a directory", cachedir) + } + + return cachedir, nil +} diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 1a072cbfa..9d89b154b 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -112,8 +112,6 @@ func newLoadBlobsProgress(s restic.Server) (*restic.Progress, error) { } } - // fmt.Printf("sec: %v, trees: %v / %v\n", sec, s.Trees, trees) - fmt.Printf("\x1b[2K\r[%s] %3.2f%% %d trees/s %d / %d trees, %d blobs ETA %s", format_duration(d), float64(s.Trees)/float64(trees)*100, diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go new file mode 100644 index 000000000..40dd929da --- /dev/null +++ b/cmd/restic/cmd_cache.go @@ -0,0 +1,114 @@ +package main + +import ( + "fmt" + "sync" + + "github.com/restic/restic" + "github.com/restic/restic/backend" +) + +type CmdCache struct{} + +func init() { + _, err := parser.AddCommand("cache", + "manage cache", + "The cache command creates and manages the local cache", + &CmdCache{}) + if err != nil { + panic(err) + } +} + +func (cmd CmdCache) Usage() string { + return "[update|clear]" +} + +func (cmd CmdCache) Execute(args []string) error { + // if len(args) == 0 || len(args) > 2 { + // return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) + // } + + s, err := OpenRepo() + if err != nil { + return err + } + + fmt.Printf("update cache, load trees\n") + + list, err := s.List(backend.Tree) + if err != nil { + return err + } + + cache, err := restic.NewCache() + if err != nil { + return err + } + + treeCh := make(chan backend.ID) + worker := func(wg *sync.WaitGroup, ch chan backend.ID) { + for treeID := range ch { + cached, err := cache.Has(backend.Tree, treeID) + if err != nil { + fmt.Printf("tree %v cache error: %v\n", treeID.Str(), err) + continue + } + + if cached { + fmt.Printf("tree %v already cached\n", treeID.Str()) + continue + } + + rd, err := s.GetReader(backend.Tree, treeID) + if err != nil { + fmt.Printf(" load error: %v\n", err) + continue + } + + decRd, err := s.Key().DecryptFrom(rd) + if err != nil { + fmt.Printf(" store error: %v\n", err) + continue + } + + err = cache.Store(backend.Tree, treeID, decRd) + if err != nil { + fmt.Printf(" store error: %v\n", err) + continue + } + + err = decRd.Close() + if err != nil { + fmt.Printf(" close error: %v\n", err) + continue + } + + err = rd.Close() + if err != nil { + fmt.Printf(" close error: %v\n", err) + continue + } + + fmt.Printf("tree %v stored\n", treeID.Str()) + } + wg.Done() + } + + var wg sync.WaitGroup + // start workers + for i := 0; i < 500; i++ { + wg.Add(1) + go worker(&wg, treeCh) + } + + for _, treeID := range list { + treeCh <- treeID + } + + close(treeCh) + + wg.Wait() + + return nil +} diff --git a/cmd/restic/main.go b/cmd/restic/main.go index b15b28a7d..df6402640 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -165,6 +165,7 @@ func init() { func main() { // defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop() + // defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop() opts.Repo = os.Getenv("RESTIC_REPOSITORY") debug.Log("restic", "main %#v", os.Args)