From c4fc7b52ae06c17545692d4fdbc2c2071c849086 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 8 Nov 2015 21:10:03 +0100 Subject: [PATCH] Add 'optimize' command that repacks blobs --- cmd/restic/cmd_optimize.go | 84 ++++++++++++++++++++++++++++++++++ cmd/restic/integration_test.go | 27 ++++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 cmd/restic/cmd_optimize.go diff --git a/cmd/restic/cmd_optimize.go b/cmd/restic/cmd_optimize.go new file mode 100644 index 000000000..1e29ce1d7 --- /dev/null +++ b/cmd/restic/cmd_optimize.go @@ -0,0 +1,84 @@ +package main + +import ( + "errors" + "fmt" + + "github.com/restic/restic/backend" + "github.com/restic/restic/checker" +) + +type CmdOptimize struct { + global *GlobalOptions +} + +func init() { + _, err := parser.AddCommand("optimize", + "optimize the repository", + "The optimize command reorganizes the repository and removes uneeded data", + &CmdOptimize{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +func (cmd CmdOptimize) Usage() string { + return "[optimize-options]" +} + +func (cmd CmdOptimize) Execute(args []string) error { + if len(args) != 0 { + return errors.New("optimize has no arguments") + } + + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + cmd.global.Verbosef("Create exclusive lock for repository\n") + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + chkr := checker.New(repo) + + cmd.global.Verbosef("Load indexes\n") + _, errs := chkr.LoadIndex() + + if len(errs) > 0 { + for _, err := range errs { + cmd.global.Warnf("error: %v\n", err) + } + return fmt.Errorf("LoadIndex returned errors") + } + + done := make(chan struct{}) + errChan := make(chan error) + go chkr.Structure(errChan, done) + + for err := range errChan { + if e, ok := err.(checker.TreeError); ok { + cmd.global.Warnf("error for tree %v:\n", e.ID.Str()) + for _, treeErr := range e.Errors { + cmd.global.Warnf(" %v\n", treeErr) + } + } else { + cmd.global.Warnf("error: %v\n", err) + } + } + + unusedBlobs := backend.NewIDSet(chkr.UnusedBlobs()...) + cmd.global.Verbosef("%d unused blobs found, repacking...\n", len(unusedBlobs)) + + repacker := checker.NewRepacker(repo, unusedBlobs) + err = repacker.Repack() + if err != nil { + return err + } + + cmd.global.Verbosef("repacking done\n") + return nil +} diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index fa95eca92..5c5a196d1 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -61,7 +61,7 @@ func cmdBackupExcludes(t testing.TB, global GlobalOptions, target []string, pare OK(t, cmd.Execute(target)) } -func cmdList(t testing.TB, global GlobalOptions, tpe string) []backend.ID { +func cmdList(t testing.TB, global GlobalOptions, tpe string) backend.IDs { var buf bytes.Buffer global.stdout = &buf cmd := &CmdList{global: &global} @@ -87,7 +87,11 @@ func cmdRestoreIncludes(t testing.TB, global GlobalOptions, dir string, snapshot } func cmdCheck(t testing.TB, global GlobalOptions) { - cmd := &CmdCheck{global: &global, ReadData: true} + cmd := &CmdCheck{ + global: &global, + ReadData: true, + CheckUnused: true, + } OK(t, cmd.Execute(nil)) } @@ -105,6 +109,11 @@ func cmdRebuildIndex(t testing.TB, global GlobalOptions) { OK(t, cmd.Execute(nil)) } +func cmdOptimize(t testing.TB, global GlobalOptions) { + cmd := &CmdOptimize{global: &global} + OK(t, cmd.Execute(nil)) +} + func cmdLs(t testing.TB, global GlobalOptions, snapshotID string) []string { var buf bytes.Buffer global.stdout = &buf @@ -689,3 +698,17 @@ func TestRebuildIndexAlwaysFull(t *testing.T) { repository.IndexFull = func(*repository.Index) bool { return true } TestRebuildIndex(t) } + +func TestOptimizeRemoveUnusedBlobs(t *testing.T) { + withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) { + datafile := filepath.Join("..", "..", "checker", "testdata", "checker-test-repo.tar.gz") + SetupTarTestFixture(t, env.base, datafile) + + // snapshotIDs := cmdList(t, global, "snapshots") + // t.Logf("snapshots: %v", snapshotIDs) + + OK(t, os.Remove(filepath.Join(env.repo, "snapshots", "a13c11e582b77a693dd75ab4e3a3ba96538a056594a4b9076e4cacebe6e06d43"))) + cmdOptimize(t, global) + cmdCheck(t, global) + }) +}