diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index e8596ae33..fc460e39e 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -296,9 +296,6 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args errorsFound = true printer.E("%v\n", err) } - } else if err == checker.ErrLegacyLayout { - errorsFound = true - printer.E("error: repository still uses the S3 legacy layout\nYou must run `restic migrate s3legacy` to correct this.\n") } else { errorsFound = true printer.E("%v\n", err) diff --git a/cmd/restic/cmd_restore_integration_test.go b/cmd/restic/cmd_restore_integration_test.go index b0543850b..42cd1f87d 100644 --- a/cmd/restic/cmd_restore_integration_test.go +++ b/cmd/restic/cmd_restore_integration_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/feature" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" "github.com/restic/restic/internal/ui/termstatus" @@ -403,36 +402,21 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) { "meta data of intermediate directory hasn't been restore") } -func TestRestoreLocalLayout(t *testing.T) { - defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() +func TestRestoreDefaultLayout(t *testing.T) { env, cleanup := withTestEnvironment(t) defer cleanup() - var tests = []struct { - filename string - layout string - }{ - {"repo-layout-default.tar.gz", ""}, - {"repo-layout-s3legacy.tar.gz", ""}, - {"repo-layout-default.tar.gz", "default"}, - {"repo-layout-s3legacy.tar.gz", "s3legacy"}, - } + datafile := filepath.Join("..", "..", "internal", "backend", "testdata", "repo-layout-default.tar.gz") - for _, test := range tests { - datafile := filepath.Join("..", "..", "internal", "backend", "testdata", test.filename) + rtest.SetupTarTestFixture(t, env.base, datafile) - rtest.SetupTarTestFixture(t, env.base, datafile) + // check the repo + testRunCheck(t, env.gopts) - env.gopts.extended["local.layout"] = test.layout + // restore latest snapshot + target := filepath.Join(env.base, "restore") + testRunRestoreLatest(t, env.gopts, target, nil, nil) - // check the repo - testRunCheck(t, env.gopts) - - // restore latest snapshot - target := filepath.Join(env.base, "restore") - testRunRestoreLatest(t, env.gopts, target, nil, nil) - - rtest.RemoveAll(t, filepath.Join(env.base, "repo")) - rtest.RemoveAll(t, target) - } + rtest.RemoveAll(t, filepath.Join(env.base, "repo")) + rtest.RemoveAll(t, target) } diff --git a/doc/design.rst b/doc/design.rst index c974e997a..62b7e9bf9 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -119,16 +119,11 @@ A local repository can be initialized with the ``restic init`` command, e.g.: $ restic -r /tmp/restic-repo init -The local and sftp backends will auto-detect and accept all layouts described -in the following sections, so that remote repositories mounted locally e.g. via -fuse can be accessed. The layout auto-detection can be overridden by specifying -the option ``-o local.layout=default``, valid values are ``default`` and -``s3legacy``. The option for the sftp backend is named ``sftp.layout``, for the -s3 backend ``s3.layout``. - S3 Legacy Layout (deprecated) ----------------------------- +Restic 0.17 is the last version that supports the legacy layout. + Unfortunately during development the Amazon S3 backend uses slightly different paths (directory names use singular instead of plural for ``key``, ``lock``, and ``snapshot`` files), and the pack files are stored directly below @@ -152,8 +147,6 @@ the ``data`` directory. The S3 Legacy repository layout looks like this: /snapshot └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec -Restic 0.17 is the last version that supports the legacy layout. - Pack Format =========== diff --git a/internal/backend/layout/layout.go b/internal/backend/layout/layout.go index 052fd66ca..cd69efc34 100644 --- a/internal/backend/layout/layout.go +++ b/internal/backend/layout/layout.go @@ -1,18 +1,7 @@ package layout import ( - "context" - "fmt" - "os" - "path/filepath" - "regexp" - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/feature" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/restic" ) // Layout computes paths for file name storage. @@ -23,159 +12,3 @@ type Layout interface { Paths() []string Name() string } - -// Filesystem is the abstraction of a file system used for a backend. -type Filesystem interface { - Join(...string) string - ReadDir(context.Context, string) ([]os.FileInfo, error) - IsNotExist(error) bool -} - -// ensure statically that *LocalFilesystem implements Filesystem. -var _ Filesystem = &LocalFilesystem{} - -// LocalFilesystem implements Filesystem in a local path. -type LocalFilesystem struct { -} - -// ReadDir returns all entries of a directory. -func (l *LocalFilesystem) ReadDir(_ context.Context, dir string) ([]os.FileInfo, error) { - f, err := fs.Open(dir) - if err != nil { - return nil, err - } - - entries, err := f.Readdir(-1) - if err != nil { - return nil, errors.Wrap(err, "Readdir") - } - - err = f.Close() - if err != nil { - return nil, errors.Wrap(err, "Close") - } - - return entries, nil -} - -// Join combines several path components to one. -func (l *LocalFilesystem) Join(paths ...string) string { - return filepath.Join(paths...) -} - -// IsNotExist returns true for errors that are caused by not existing files. -func (l *LocalFilesystem) IsNotExist(err error) bool { - return os.IsNotExist(err) -} - -var backendFilenameLength = len(restic.ID{}) * 2 -var backendFilename = regexp.MustCompile(fmt.Sprintf("^[a-fA-F0-9]{%d}$", backendFilenameLength)) - -func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error) { - entries, err := fs.ReadDir(ctx, dir) - if err != nil && fs.IsNotExist(err) { - return false, nil - } - - if err != nil { - return false, errors.Wrap(err, "ReadDir") - } - - for _, e := range entries { - if backendFilename.MatchString(e.Name()) { - return true, nil - } - } - - return false, nil -} - -// ErrLayoutDetectionFailed is returned by DetectLayout() when the layout -// cannot be detected automatically. -var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed") - -var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository") - -// DetectLayout tries to find out which layout is used in a local (or sftp) -// filesystem at the given path. If repo is nil, an instance of LocalFilesystem -// is used. -func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, error) { - debug.Log("detect layout at %v", dir) - if repo == nil { - repo = &LocalFilesystem{} - } - - // key file in the "keys" dir (DefaultLayout) - foundKeysFile, err := hasBackendFile(ctx, repo, repo.Join(dir, defaultLayoutPaths[backend.KeyFile])) - if err != nil { - return nil, err - } - - // key file in the "key" dir (S3LegacyLayout) - foundKeyFile, err := hasBackendFile(ctx, repo, repo.Join(dir, s3LayoutPaths[backend.KeyFile])) - if err != nil { - return nil, err - } - - if foundKeysFile && !foundKeyFile { - debug.Log("found default layout at %v", dir) - return &DefaultLayout{ - Path: dir, - Join: repo.Join, - }, nil - } - - if foundKeyFile && !foundKeysFile { - if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) { - return nil, ErrLegacyLayoutFound - } - - debug.Log("found s3 layout at %v", dir) - return &S3LegacyLayout{ - Path: dir, - Join: repo.Join, - }, nil - } - - debug.Log("layout detection failed") - return nil, ErrLayoutDetectionFailed -} - -// ParseLayout parses the config string and returns a Layout. When layout is -// the empty string, DetectLayout is used. If that fails, defaultLayout is used. -func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, path string) (l Layout, err error) { - debug.Log("parse layout string %q for backend at %v", layout, path) - switch layout { - case "default": - l = &DefaultLayout{ - Path: path, - Join: repo.Join, - } - case "s3legacy": - if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) { - return nil, ErrLegacyLayoutFound - } - - l = &S3LegacyLayout{ - Path: path, - Join: repo.Join, - } - case "": - l, err = DetectLayout(ctx, repo, path) - - // use the default layout if auto detection failed - if errors.Is(err, ErrLayoutDetectionFailed) && defaultLayout != "" { - debug.Log("error: %v, use default layout %v", err, defaultLayout) - return ParseLayout(ctx, repo, defaultLayout, "", path) - } - - if err != nil { - return nil, err - } - debug.Log("layout detected: %v", l) - default: - return nil, errors.Errorf("unknown backend layout string %q, may be one of: default, s3legacy", layout) - } - - return l, nil -} diff --git a/internal/backend/layout/layout_default.go b/internal/backend/layout/layout_default.go index 9a8419f10..3f73a941d 100644 --- a/internal/backend/layout/layout_default.go +++ b/internal/backend/layout/layout_default.go @@ -23,6 +23,13 @@ var defaultLayoutPaths = map[backend.FileType]string{ backend.KeyFile: "keys", } +func NewDefaultLayout(path string, join func(...string) string) *DefaultLayout { + return &DefaultLayout{ + Path: path, + Join: join, + } +} + func (l *DefaultLayout) String() string { return "" } diff --git a/internal/backend/layout/layout_s3legacy.go b/internal/backend/layout/layout_s3legacy.go deleted file mode 100644 index 8b90789d8..000000000 --- a/internal/backend/layout/layout_s3legacy.go +++ /dev/null @@ -1,79 +0,0 @@ -package layout - -import ( - "github.com/restic/restic/internal/backend" -) - -// S3LegacyLayout implements the old layout used for s3 cloud storage backends, as -// described in the Design document. -type S3LegacyLayout struct { - URL string - Path string - Join func(...string) string -} - -var s3LayoutPaths = map[backend.FileType]string{ - backend.PackFile: "data", - backend.SnapshotFile: "snapshot", - backend.IndexFile: "index", - backend.LockFile: "lock", - backend.KeyFile: "key", -} - -func (l *S3LegacyLayout) String() string { - return "" -} - -// Name returns the name for this layout. -func (l *S3LegacyLayout) Name() string { - return "s3legacy" -} - -// join calls Join with the first empty elements removed. -func (l *S3LegacyLayout) join(url string, items ...string) string { - for len(items) > 0 && items[0] == "" { - items = items[1:] - } - - path := l.Join(items...) - if path == "" || path[0] != '/' { - if url != "" && url[len(url)-1] != '/' { - url += "/" - } - } - - return url + path -} - -// Dirname returns the directory path for a given file type and name. -func (l *S3LegacyLayout) Dirname(h backend.Handle) string { - if h.Type == backend.ConfigFile { - return l.URL + l.Join(l.Path, "/") - } - - return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/" -} - -// Filename returns a path to a file, including its name. -func (l *S3LegacyLayout) Filename(h backend.Handle) string { - name := h.Name - - if h.Type == backend.ConfigFile { - name = "config" - } - - return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name) -} - -// Paths returns all directory names -func (l *S3LegacyLayout) Paths() (dirs []string) { - for _, p := range s3LayoutPaths { - dirs = append(dirs, l.Join(l.Path, p)) - } - return dirs -} - -// Basedir returns the base dir name for type t. -func (l *S3LegacyLayout) Basedir(t backend.FileType) (dirname string, subdirs bool) { - return l.Join(l.Path, s3LayoutPaths[t]), false -} diff --git a/internal/backend/layout/layout_test.go b/internal/backend/layout/layout_test.go index 55a0749c9..de5ae7d69 100644 --- a/internal/backend/layout/layout_test.go +++ b/internal/backend/layout/layout_test.go @@ -1,7 +1,6 @@ package layout import ( - "context" "fmt" "path" "path/filepath" @@ -10,7 +9,6 @@ import ( "testing" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) @@ -232,42 +230,6 @@ func TestRESTLayoutURLs(t *testing.T) { "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, - { - &S3LegacyLayout{URL: "https://hostname.foo", Path: "/", Join: path.Join}, - backend.Handle{Type: backend.PackFile, Name: "foobar"}, - "https://hostname.foo/data/foobar", - "https://hostname.foo/data/", - }, - { - &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "", Join: path.Join}, - backend.Handle{Type: backend.LockFile, Name: "foobar"}, - "https://hostname.foo:1234/prefix/repo/lock/foobar", - "https://hostname.foo:1234/prefix/repo/lock/", - }, - { - &S3LegacyLayout{URL: "https://hostname.foo:1234/prefix/repo", Path: "/", Join: path.Join}, - backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, - "https://hostname.foo:1234/prefix/repo/config", - "https://hostname.foo:1234/prefix/repo/", - }, - { - &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - backend.Handle{Type: backend.PackFile, Name: "foobar"}, - "data/foobar", - "data/", - }, - { - &S3LegacyLayout{URL: "", Path: "", Join: path.Join}, - backend.Handle{Type: backend.LockFile, Name: "foobar"}, - "lock/foobar", - "lock/", - }, - { - &S3LegacyLayout{URL: "", Path: "/", Join: path.Join}, - backend.Handle{Type: backend.ConfigFile, Name: "foobar"}, - "/config", - "/", - }, } for _, test := range tests { @@ -284,165 +246,3 @@ func TestRESTLayoutURLs(t *testing.T) { }) } } - -func TestS3LegacyLayout(t *testing.T) { - path := rtest.TempDir(t) - - var tests = []struct { - backend.Handle - filename string - }{ - { - backend.Handle{Type: backend.PackFile, Name: "0123456"}, - filepath.Join(path, "data", "0123456"), - }, - { - backend.Handle{Type: backend.ConfigFile, Name: "CFG"}, - filepath.Join(path, "config"), - }, - { - backend.Handle{Type: backend.SnapshotFile, Name: "123456"}, - filepath.Join(path, "snapshot", "123456"), - }, - { - backend.Handle{Type: backend.IndexFile, Name: "123456"}, - filepath.Join(path, "index", "123456"), - }, - { - backend.Handle{Type: backend.LockFile, Name: "123456"}, - filepath.Join(path, "lock", "123456"), - }, - { - backend.Handle{Type: backend.KeyFile, Name: "123456"}, - filepath.Join(path, "key", "123456"), - }, - } - - l := &S3LegacyLayout{ - Path: path, - Join: filepath.Join, - } - - t.Run("Paths", func(t *testing.T) { - dirs := l.Paths() - - want := []string{ - filepath.Join(path, "data"), - filepath.Join(path, "snapshot"), - filepath.Join(path, "index"), - filepath.Join(path, "lock"), - filepath.Join(path, "key"), - } - - sort.Strings(want) - sort.Strings(dirs) - - if !reflect.DeepEqual(dirs, want) { - t.Fatalf("wrong paths returned, want:\n %v\ngot:\n %v", want, dirs) - } - }) - - for _, test := range tests { - t.Run(fmt.Sprintf("%v/%v", test.Type, test.Handle.Name), func(t *testing.T) { - filename := l.Filename(test.Handle) - if filename != test.filename { - t.Fatalf("wrong filename, want %v, got %v", test.filename, filename) - } - }) - } -} - -func TestDetectLayout(t *testing.T) { - defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() - path := rtest.TempDir(t) - - var tests = []struct { - filename string - want string - }{ - {"repo-layout-default.tar.gz", "*layout.DefaultLayout"}, - {"repo-layout-s3legacy.tar.gz", "*layout.S3LegacyLayout"}, - } - - var fs = &LocalFilesystem{} - for _, test := range tests { - for _, fs := range []Filesystem{fs, nil} { - t.Run(fmt.Sprintf("%v/fs-%T", test.filename, fs), func(t *testing.T) { - rtest.SetupTarTestFixture(t, path, filepath.Join("../testdata", test.filename)) - - layout, err := DetectLayout(context.TODO(), fs, filepath.Join(path, "repo")) - if err != nil { - t.Fatal(err) - } - - if layout == nil { - t.Fatal("wanted some layout, but detect returned nil") - } - - layoutName := fmt.Sprintf("%T", layout) - if layoutName != test.want { - t.Fatalf("want layout %v, got %v", test.want, layoutName) - } - - rtest.RemoveAll(t, filepath.Join(path, "repo")) - }) - } - } -} - -func TestParseLayout(t *testing.T) { - defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() - path := rtest.TempDir(t) - - var tests = []struct { - layoutName string - defaultLayoutName string - want string - }{ - {"default", "", "*layout.DefaultLayout"}, - {"s3legacy", "", "*layout.S3LegacyLayout"}, - {"", "", "*layout.DefaultLayout"}, - } - - rtest.SetupTarTestFixture(t, path, filepath.Join("..", "testdata", "repo-layout-default.tar.gz")) - - for _, test := range tests { - t.Run(test.layoutName, func(t *testing.T) { - layout, err := ParseLayout(context.TODO(), &LocalFilesystem{}, test.layoutName, test.defaultLayoutName, filepath.Join(path, "repo")) - if err != nil { - t.Fatal(err) - } - - if layout == nil { - t.Fatal("wanted some layout, but detect returned nil") - } - - // test that the functions work (and don't panic) - _ = layout.Dirname(backend.Handle{Type: backend.PackFile}) - _ = layout.Filename(backend.Handle{Type: backend.PackFile, Name: "1234"}) - _ = layout.Paths() - - layoutName := fmt.Sprintf("%T", layout) - if layoutName != test.want { - t.Fatalf("want layout %v, got %v", test.want, layoutName) - } - }) - } -} - -func TestParseLayoutInvalid(t *testing.T) { - path := rtest.TempDir(t) - - var invalidNames = []string{ - "foo", "bar", "local", - } - - for _, name := range invalidNames { - t.Run(name, func(t *testing.T) { - layout, err := ParseLayout(context.TODO(), nil, name, "", path) - if err == nil { - t.Fatalf("expected error not found for layout name %v, layout is %v", name, layout) - } - }) - } -} diff --git a/internal/backend/local/config.go b/internal/backend/local/config.go index e08f05550..782f132d0 100644 --- a/internal/backend/local/config.go +++ b/internal/backend/local/config.go @@ -9,8 +9,7 @@ import ( // Config holds all information needed to open a local repository. type Config struct { - Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"` + Path string Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"` } diff --git a/internal/backend/local/layout_test.go b/internal/backend/local/layout_test.go index 00c91376a..cac89e552 100644 --- a/internal/backend/local/layout_test.go +++ b/internal/backend/local/layout_test.go @@ -6,30 +6,22 @@ import ( "testing" "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) func TestLayout(t *testing.T) { - defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { filename string - layout string failureExpected bool packfiles map[string]bool }{ - {"repo-layout-default.tar.gz", "", false, map[string]bool{ + {"repo-layout-default.tar.gz", false, map[string]bool{ "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, }}, - {"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{ - "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, - "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, - "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, - }}, } for _, test := range tests { @@ -39,7 +31,6 @@ func TestLayout(t *testing.T) { repo := filepath.Join(path, "repo") be, err := Open(context.TODO(), Config{ Path: repo, - Layout: test.layout, Connections: 2, }) if err != nil { diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index f041d608a..ff7e3d35d 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -37,13 +37,8 @@ func NewFactory() location.Factory { return location.NewLimitedBackendFactory("local", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open)) } -const defaultLayout = "default" - func open(ctx context.Context, cfg Config) (*Local, error) { - l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path) - if err != nil { - return nil, err - } + l := layout.NewDefaultLayout(cfg.Path, filepath.Join) fi, err := fs.Stat(l.Filename(backend.Handle{Type: backend.ConfigFile})) m := util.DeriveModesFromFileInfo(fi, err) @@ -58,14 +53,14 @@ func open(ctx context.Context, cfg Config) (*Local, error) { // Open opens the local backend as specified by config. func Open(ctx context.Context, cfg Config) (*Local, error) { - debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout) + debug.Log("open local backend at %v", cfg.Path) return open(ctx, cfg) } // Create creates all the necessary files and directories for a new local // backend at dir. Afterwards a new config blob should be created. func Create(ctx context.Context, cfg Config) (*Local, error) { - debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout) + debug.Log("create local backend at %v", cfg.Path) be, err := open(ctx, cfg) if err != nil { diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 019f8471b..5ef952891 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -37,8 +37,6 @@ func NewFactory() location.Factory { return location.NewHTTPBackendFactory("s3", ParseConfig, location.NoPassword, Create, Open) } -const defaultLayout = "default" - func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) @@ -83,15 +81,9 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro be := &Backend{ client: client, cfg: cfg, + Layout: layout.NewDefaultLayout(cfg.Prefix, path.Join), } - l, err := layout.ParseLayout(ctx, be, cfg.Layout, defaultLayout, cfg.Prefix) - if err != nil { - return nil, err - } - - be.Layout = l - return be, nil } diff --git a/internal/backend/sftp/config.go b/internal/backend/sftp/config.go index aa8ac7bff..daefbf441 100644 --- a/internal/backend/sftp/config.go +++ b/internal/backend/sftp/config.go @@ -13,7 +13,6 @@ import ( type Config struct { User, Host, Port, Path string - Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"` Command string `option:"command" help:"specify command to create sftp connection"` Args string `option:"args" help:"specify arguments for ssh"` diff --git a/internal/backend/sftp/layout_test.go b/internal/backend/sftp/layout_test.go index 8bb7eac01..9e143d4fd 100644 --- a/internal/backend/sftp/layout_test.go +++ b/internal/backend/sftp/layout_test.go @@ -8,7 +8,6 @@ import ( "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/sftp" - "github.com/restic/restic/internal/feature" rtest "github.com/restic/restic/internal/test" ) @@ -17,25 +16,18 @@ func TestLayout(t *testing.T) { t.Skip("sftp server binary not available") } - defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)() path := rtest.TempDir(t) var tests = []struct { filename string - layout string failureExpected bool packfiles map[string]bool }{ - {"repo-layout-default.tar.gz", "", false, map[string]bool{ + {"repo-layout-default.tar.gz", false, map[string]bool{ "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, }}, - {"repo-layout-s3legacy.tar.gz", "", false, map[string]bool{ - "fc919a3b421850f6fa66ad22ebcf91e433e79ffef25becf8aef7c7b1eca91683": false, - "c089d62788da14f8b7cbf77188305c0874906f0b73d3fce5a8869050e8d0c0e1": false, - "aa464e9fd598fe4202492ee317ffa728e82fa83a1de1a61996e5bd2d6651646c": false, - }}, } for _, test := range tests { @@ -46,7 +38,6 @@ func TestLayout(t *testing.T) { be, err := sftp.Open(context.TODO(), sftp.Config{ Command: fmt.Sprintf("%q -e", sftpServer), Path: repo, - Layout: test.layout, Connections: 5, }) if err != nil { diff --git a/internal/backend/sftp/sftp.go b/internal/backend/sftp/sftp.go index d766591b7..8ac6781e9 100644 --- a/internal/backend/sftp/sftp.go +++ b/internal/backend/sftp/sftp.go @@ -121,7 +121,13 @@ func startClient(cfg Config) (*SFTP, error) { } _, posixRename := client.HasExtension("posix-rename@openssh.com") - return &SFTP{c: client, cmd: cmd, result: ch, posixRename: posixRename}, nil + return &SFTP{ + c: client, + cmd: cmd, + result: ch, + posixRename: posixRename, + Layout: layout.NewDefaultLayout(cfg.Path, path.Join), + }, nil } // clientError returns an error if the client has exited. Otherwise, nil is @@ -152,14 +158,6 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) { } func open(ctx context.Context, sftp *SFTP, cfg Config) (*SFTP, error) { - var err error - sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) - if err != nil { - return nil, err - } - - debug.Log("layout: %v\n", sftp.Layout) - fi, err := sftp.c.Stat(sftp.Layout.Filename(backend.Handle{Type: backend.ConfigFile})) m := util.DeriveModesFromFileInfo(fi, err) debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir) @@ -195,11 +193,6 @@ func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error { return g.Wait() } -// Join combines path components with slashes (according to the sftp spec). -func (r *SFTP) Join(p ...string) string { - return path.Join(p...) -} - // ReadDir returns the entries for a directory. func (r *SFTP) ReadDir(_ context.Context, dir string) ([]os.FileInfo, error) { fi, err := r.c.ReadDir(dir) @@ -266,11 +259,6 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) { return nil, err } - sftp.Layout, err = layout.ParseLayout(ctx, sftp, cfg.Layout, defaultLayout, cfg.Path) - if err != nil { - return nil, err - } - sftp.Modes = util.DefaultModes // test if config file already exists @@ -582,7 +570,7 @@ func (r *SFTP) deleteRecursive(ctx context.Context, name string) error { return ctx.Err() } - itemName := r.Join(name, fi.Name()) + itemName := path.Join(name, fi.Name()) if fi.IsDir() { err := r.deleteRecursive(ctx, itemName) if err != nil { diff --git a/internal/backend/testdata/repo-layout-s3legacy.tar.gz b/internal/backend/testdata/repo-layout-s3legacy.tar.gz deleted file mode 100644 index 2b7d852cc..000000000 Binary files a/internal/backend/testdata/repo-layout-s3legacy.tar.gz and /dev/null differ diff --git a/internal/checker/checker.go b/internal/checker/checker.go index e0c1766d7..76bb15f63 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -8,8 +8,6 @@ import ( "sync" "github.com/klauspost/compress/zstd" - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/s3" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -53,9 +51,6 @@ func New(repo restic.Repository, trackUnused bool) *Checker { return c } -// ErrLegacyLayout is returned when the repository uses the S3 legacy layout. -var ErrLegacyLayout = errors.New("repository uses S3 legacy layout") - // ErrDuplicatePacks is returned when a pack is found in more than one index. type ErrDuplicatePacks struct { PackID restic.ID @@ -177,23 +172,11 @@ func (e *PackError) Error() string { return "pack " + e.ID.String() + ": " + e.Err.Error() } -func isS3Legacy(b backend.Backend) bool { - be := backend.AsBackend[*s3.Backend](b) - return be != nil && be.Layout.Name() == "s3legacy" -} - // Packs checks that all packs referenced in the index are still available and // there are no packs that aren't in an index. errChan is closed after all // packs have been checked. func (c *Checker) Packs(ctx context.Context, errChan chan<- error) { defer close(errChan) - - if r, ok := c.repo.(*repository.Repository); ok { - if isS3Legacy(repository.AsS3Backend(r)) { - errChan <- ErrLegacyLayout - } - } - debug.Log("checking for %d packs", len(c.packs)) debug.Log("listing repository packs") diff --git a/internal/feature/registry.go b/internal/feature/registry.go index 8bdb5480e..7fe7da965 100644 --- a/internal/feature/registry.go +++ b/internal/feature/registry.go @@ -6,7 +6,6 @@ var Flag = New() // flag names are written in kebab-case const ( BackendErrorRedesign FlagName = "backend-error-redesign" - DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout" DeviceIDForHardlinks FlagName = "device-id-for-hardlinks" ExplicitS3AnonymousAuth FlagName = "explicit-s3-anonymous-auth" SafeForgetKeepTags FlagName = "safe-forget-keep-tags" @@ -15,7 +14,6 @@ const ( func init() { Flag.SetFlags(map[FlagName]FlagDesc{ BackendErrorRedesign: {Type: Beta, Description: "enforce timeouts for stuck HTTP requests and use new backend error handling design."}, - DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."}, DeviceIDForHardlinks: {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"}, ExplicitS3AnonymousAuth: {Type: Beta, Description: "forbid anonymous S3 authentication unless `-o s3.unsafe-anonymous-auth=true` is set"}, SafeForgetKeepTags: {Type: Beta, Description: "prevent deleting all snapshots if the tag passed to `forget --keep-tags tagname` does not exist"}, diff --git a/internal/migrations/s3_layout.go b/internal/migrations/s3_layout.go deleted file mode 100644 index 8b994b8fc..000000000 --- a/internal/migrations/s3_layout.go +++ /dev/null @@ -1,123 +0,0 @@ -package migrations - -import ( - "context" - "fmt" - "os" - "path" - - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/layout" - "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/debug" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" -) - -func init() { - register(&S3Layout{}) -} - -// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the -// "default" layout. -type S3Layout struct{} - -// Check tests whether the migration can be applied. -func (m *S3Layout) Check(_ context.Context, repo restic.Repository) (bool, string, error) { - be := repository.AsS3Backend(repo.(*repository.Repository)) - if be == nil { - debug.Log("backend is not s3") - return false, "backend is not s3", nil - } - - if be.Layout.Name() != "s3legacy" { - debug.Log("layout is not s3legacy") - return false, "not using the legacy s3 layout", nil - } - - return true, "", nil -} - -func (m *S3Layout) RepoCheck() bool { - return false -} - -func retry(max int, fail func(err error), f func() error) error { - var err error - for i := 0; i < max; i++ { - err = f() - if err == nil { - return nil - } - if fail != nil { - fail(err) - } - } - return err -} - -// maxErrors for retrying renames on s3. -const maxErrors = 20 - -func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l layout.Layout, t restic.FileType) error { - printErr := func(err error) { - fmt.Fprintf(os.Stderr, "renaming file returned error: %v\n", err) - } - - return be.List(ctx, t, func(fi backend.FileInfo) error { - h := backend.Handle{Type: t, Name: fi.Name} - debug.Log("move %v", h) - - return retry(maxErrors, printErr, func() error { - return be.Rename(ctx, h, l) - }) - }) -} - -// Apply runs the migration. -func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { - be := repository.AsS3Backend(repo.(*repository.Repository)) - if be == nil { - debug.Log("backend is not s3") - return errors.New("backend is not s3") - } - - oldLayout := &layout.S3LegacyLayout{ - Path: be.Path(), - Join: path.Join, - } - - newLayout := &layout.DefaultLayout{ - Path: be.Path(), - Join: path.Join, - } - - be.Layout = oldLayout - - for _, t := range []restic.FileType{ - restic.SnapshotFile, - restic.PackFile, - restic.KeyFile, - restic.LockFile, - } { - err := m.moveFiles(ctx, be, newLayout, t) - if err != nil { - return err - } - } - - be.Layout = newLayout - - return nil -} - -// Name returns the name for this migration. -func (m *S3Layout) Name() string { - return "s3_layout" -} - -// Desc returns a short description what the migration does. -func (m *S3Layout) Desc() string { - return "move files from 's3legacy' to the 'default' repository layout" -} diff --git a/internal/repository/s3_backend.go b/internal/repository/s3_backend.go deleted file mode 100644 index 4c77c69a2..000000000 --- a/internal/repository/s3_backend.go +++ /dev/null @@ -1,12 +0,0 @@ -package repository - -import ( - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/backend/s3" -) - -// AsS3Backend extracts the S3 backend from a repository -// TODO remove me once restic 0.17 was released -func AsS3Backend(repo *Repository) *s3.Backend { - return backend.AsBackend[*s3.Backend](repo.be) -}