Merge pull request #1529 from restic/fix-1528

Fix creating missing dirs in local/sftp repo

internal/backend/local/local.go

@@ -52,27 +52,7 @@ func Open(cfg Config) (*Local, error) {
 		return nil, err
 	}
 
-	be := &Local{Config: cfg, Layout: l}
-
-	// if data dir exists, make sure that all subdirs also exist
-	datadir := be.Dirname(restic.Handle{Type: restic.DataFile})
-	if dirExists(datadir) {
-		debug.Log("datadir %v exists", datadir)
-		for _, d := range be.Paths() {
-			if !fs.HasPathPrefix(datadir, d) {
-				debug.Log("%v is not subdir of datadir %v", d, datadir)
-				continue
-			}
-
-			debug.Log("MkdirAll %v", d)
-			err := fs.MkdirAll(d, backend.Modes.Dir)
-			if err != nil {
-				return nil, errors.Wrap(err, "MkdirAll")
-			}
-		}
-	}
-
-	return be, nil
+	return &Local{Config: cfg, Layout: l}, nil
 }
 
 // Create creates all the necessary files and directories for a new local
@@ -124,20 +104,24 @@ func (b *Local) Save(ctx context.Context, h restic.Handle, rd io.Reader) error {
 		return err
 	}
 
-	if h.Type == restic.LockFile {
-		lockDir := b.Dirname(h)
-		if !dirExists(lockDir) {
-			debug.Log("locks/ does not exist yet, creating now.")
-			if err := fs.MkdirAll(lockDir, backend.Modes.Dir); err != nil {
-				return errors.Wrap(err, "MkdirAll")
-			}
-		}
-	}
-
 	filename := b.Filename(h)
 
 	// create new file
 	f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
+
+	if b.IsNotExist(err) {
+		debug.Log("error %v: creating dir", err)
+
+		// error is caused by a missing directory, try to create it
+		mkdirErr := os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
+		if mkdirErr != nil {
+			debug.Log("error creating dir %v: %v", filepath.Dir(filename), mkdirErr)
+		} else {
+			// try again
+			f, err = fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
+		}
+	}
+
 	if err != nil {
 		return errors.Wrap(err, "OpenFile")
 	}
@@ -254,7 +238,7 @@ func (b *Local) List(ctx context.Context, t restic.FileType) <-chan string {
 
 		basedir, subdirs := b.Basedir(t)
 		err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error {
-			debug.Log("walk on %v, %v\n", path, fi.IsDir())
+			debug.Log("walk on %v\n", path)
 			if err != nil {
 				return err
 			}

internal/backend/sftp/sftp.go

@@ -132,48 +132,11 @@ func Open(cfg Config, preExec, postExec func()) (*SFTP, error) {
 
 	debug.Log("layout: %v\n", sftp.Layout)
 
-	if err := sftp.checkDataSubdirs(); err != nil {
-		debug.Log("checkDataSubdirs returned %v", err)
-		return nil, err
-	}
-
 	sftp.Config = cfg
 	sftp.p = cfg.Path
 	return sftp, nil
 }
 
-func (r *SFTP) checkDataSubdirs() error {
-	datadir := r.Dirname(restic.Handle{Type: restic.DataFile})
-
-	// check if all paths for data/ exist
-	entries, err := r.ReadDir(datadir)
-	if r.IsNotExist(err) {
-		return nil
-	}
-
-	if err != nil {
-		return err
-	}
-
-	subdirs := make(map[string]struct{}, len(entries))
-	for _, entry := range entries {
-		subdirs[entry.Name()] = struct{}{}
-	}
-
-	for i := 0; i < 256; i++ {
-		subdir := fmt.Sprintf("%02x", i)
-		if _, ok := subdirs[subdir]; !ok {
-			debug.Log("subdir %v is missing, creating", subdir)
-			err := r.mkdirAll(path.Join(datadir, subdir), backend.Modes.Dir)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
 func (r *SFTP) mkdirAllDataSubdirs() error {
 	for _, d := range r.Paths() {
 		err := r.mkdirAll(d, backend.Modes.Dir)
@@ -203,6 +166,8 @@ func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
 
 // IsNotExist returns true if the error is caused by a not existing file.
 func (r *SFTP) IsNotExist(err error) bool {
+	err = errors.Cause(err)
+
 	if os.IsNotExist(err) {
 		return true
 	}
@@ -334,14 +299,16 @@ func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err err
 
 	// create new file
 	f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
-	if r.IsNotExist(errors.Cause(err)) {
-		// create the locks dir, then try again
-		err = r.mkdirAll(r.Dirname(h), backend.Modes.Dir)
-		if err != nil {
-			return errors.Wrap(err, "MkdirAll")
-		}
 
-		return r.Save(ctx, h, rd)
+	if r.IsNotExist(err) {
+		// error is caused by a missing directory, try to create it
+		mkdirErr := r.mkdirAll(r.Dirname(h), backend.Modes.Dir)
+		if mkdirErr != nil {
+			debug.Log("error creating dir %v: %v", r.Dirname(h), mkdirErr)
+		} else {
+			// try again
+			f, err = r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
+		}
 	}
 
 	if err != nil {

internal/fs/path_prefix.go

@@ -5,14 +5,15 @@ import (
 )
 
 // HasPathPrefix returns true if p is a subdir of (or a file within) base. It
-// assumes a file system which is case sensitive. For relative paths, false is
-// returned.
+// assumes a file system which is case sensitive. If the paths are not of the
+// same type (one is relative, the other is absolute), false is returned.
 func HasPathPrefix(base, p string) bool {
 	if filepath.VolumeName(base) != filepath.VolumeName(p) {
 		return false
 	}
 
-	if !filepath.IsAbs(base) || !filepath.IsAbs(p) {
+	// handle case when base and p are not of the same type
+	if filepath.IsAbs(base) != filepath.IsAbs(p) {
 		return false
 	}
 

internal/fs/path_prefix_test.go

@@ -21,7 +21,10 @@ func TestHasPathPrefix(t *testing.T) {
 		base, p string
 		result  bool
 	}{
-		{"", "", false},
+		{"", "", true},
+		{".", ".", true},
+		{".", "foo", true},
+		{"foo", ".", false},
 		{"/", "", false},
 		{"/", "x", false},
 		{"x", "/", false},
@@ -36,6 +39,10 @@ func TestHasPathPrefix(t *testing.T) {
 		{"/home/user/foo", "/home/user/foobar", false},
 		{"/home/user/Foo", "/home/user/foo/bar/baz", false},
 		{"/home/user/foo", "/home/user/Foo/bar/baz", false},
+		{"user/foo", "user/foo/bar/baz", true},
+		{"user/foo", "./user/foo", true},
+		{"user/foo", "./user/foo/", true},
+		{"/home/user/foo", "./user/foo/", false},
 	}
 
 	for _, test := range tests {