diff --git a/CHANGELOG.md b/CHANGELOG.md index 807613034..cff38c8b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ released version of restic from the perspective of the user. Important Changes in 0.X.Y ========================== + * The `migrate` command for chaning the `s3legacy` layout to the `default` + layout for s3 backends has been improved: It can now be restarted with + `restic migrate --force s3_layout` and automatically retries operations on + error. + https://github.com/restic/restic/issues/1073 + https://github.com/restic/restic/pull/1075 Important Changes in 0.7.0 ========================== diff --git a/src/cmds/restic/cmd_migrate.go b/src/cmds/restic/cmd_migrate.go index b585534a2..8fb8b320b 100644 --- a/src/cmds/restic/cmd_migrate.go +++ b/src/cmds/restic/cmd_migrate.go @@ -21,12 +21,15 @@ name is explicitely given, a list of migrations that can be applied is printed. // MigrateOptions bundles all options for the 'check' command. type MigrateOptions struct { + Force bool } var migrateOptions MigrateOptions func init() { cmdRoot.AddCommand(cmdMigrate) + f := cmdMigrate.Flags() + f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`) } func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error { @@ -59,8 +62,12 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos } if !ok { - Warnf("migration %v cannot be applied: check failed\n", m.Name()) - continue + if !opts.Force { + Warnf("migration %v cannot be applied: check failed\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name()) + continue + } + + Warnf("check for migration %v failed, continuing anyway\n", m.Name()) } Printf("applying migration %v...\n", m.Name()) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index cb3aaf5e1..2de495403 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -453,10 +453,20 @@ func (be *Backend) Rename(h restic.Handle, l backend.Layout) error { oldname := be.Filename(h) newname := l.Filename(h) + if oldname == newname { + debug.Log(" %v is already renamed", newname) + return nil + } + debug.Log(" %v -> %v", oldname, newname) coreClient := minio.Core{Client: be.client} err := coreClient.CopyObject(be.cfg.Bucket, newname, path.Join(be.cfg.Bucket, oldname), minio.CopyConditions{}) + if err != nil && be.IsNotExist(err) { + debug.Log("copy failed: %v, seems to already have been renamed", err) + return nil + } + if err != nil { debug.Log("copy failed: %v", err) return err diff --git a/src/restic/migrations/s3_layout.go b/src/restic/migrations/s3_layout.go index 75a83b885..3ddb6ed3d 100644 --- a/src/restic/migrations/s3_layout.go +++ b/src/restic/migrations/s3_layout.go @@ -2,6 +2,8 @@ package migrations import ( "context" + "fmt" + "os" "path" "restic" "restic/backend" @@ -34,13 +36,35 @@ func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, err return true, nil } +func retry(max int, fail func(err error), f func() error) error { + var err error + for i := 0; i < max; i++ { + err = f() + if err == nil { + return err + } + if fail != nil { + fail(err) + } + } + return err +} + +// maxErrors for retrying renames on s3. +const maxErrors = 20 + func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error { + printErr := func(err error) { + fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) + } + for name := range be.List(ctx, t) { h := restic.Handle{Type: t, Name: name} debug.Log("move %v", h) - if err := be.Rename(h, l); err != nil { - return err - } + + retry(maxErrors, printErr, func() error { + return be.Rename(h, l) + }) } return nil @@ -54,15 +78,22 @@ func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { return errors.New("backend is not s3") } + oldLayout := &backend.S3LegacyLayout{ + Path: be.Path(), + Join: path.Join, + } + newLayout := &backend.DefaultLayout{ Path: be.Path(), Join: path.Join, } + be.Layout = oldLayout + for _, t := range []restic.FileType{ - restic.KeyFile, restic.SnapshotFile, restic.DataFile, + restic.KeyFile, restic.LockFile, } { err := m.moveFiles(ctx, be, newLayout, t)