From 4ad33d3c3b5a08357372054b5a510a7aaecd8316 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 16 Jul 2017 15:10:06 +0200 Subject: [PATCH 1/2] sftp: Improve check for data subdirs --- src/restic/backend/sftp/sftp.go | 58 +++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index ae2d8f8c2..3d4203162 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -126,13 +126,9 @@ func Open(cfg Config) (*SFTP, error) { debug.Log("layout: %v\n", sftp.Layout) - // create paths for data and refs. mkdirAll does nothing if the paths already exist. - for _, d := range sftp.Paths() { - err = sftp.mkdirAll(d, backend.Modes.Dir) - debug.Log("mkdirAll %v -> %v", d, err) - if err != nil { - return nil, err - } + if err := sftp.checkDataSubdirs(); err != nil { + debug.Log("checkDataSubdirs returned %v", err) + return nil, err } sftp.Config = cfg @@ -140,6 +136,46 @@ func Open(cfg Config) (*SFTP, error) { return sftp, nil } +func (r *SFTP) checkDataSubdirs() error { + datadir := r.Dirname(restic.Handle{Type: restic.DataFile}) + + // check if all paths for data/ exist + entries, err := r.c.ReadDir(datadir) + if err != nil { + return err + } + + subdirs := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + subdirs[entry.Name()] = struct{}{} + } + + for i := 0; i < 256; i++ { + subdir := fmt.Sprintf("%02x", i) + if _, ok := subdirs[subdir]; !ok { + debug.Log("subdir %v is missing, creating", subdir) + err := r.mkdirAll(path.Join(datadir, subdir), backend.Modes.Dir) + if err != nil { + return err + } + } + } + + return nil +} + +func (r *SFTP) mkdirAllDataSubdirs() error { + for _, d := range r.Paths() { + err := r.mkdirAll(d, backend.Modes.Dir) + debug.Log("mkdirAll %v -> %v", d, err) + if err != nil { + return err + } + } + + return nil +} + // Join combines path components with slashes (according to the sftp spec). func (r *SFTP) Join(p ...string) string { return path.Join(p...) @@ -211,12 +247,8 @@ func Create(cfg Config) (*SFTP, error) { } // create paths for data and refs - for _, d := range sftp.Paths() { - err = sftp.mkdirAll(d, backend.Modes.Dir) - debug.Log("mkdirAll %v -> %v", d, err) - if err != nil { - return nil, err - } + if err = sftp.mkdirAllDataSubdirs(); err != nil { + return nil, err } err = sftp.Close() From 7cdcaadcf512d798ed7f19ff9df945047f0985b3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 16 Jul 2017 15:11:26 +0200 Subject: [PATCH 2/2] Add entry to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c7dab846..eb76e9337 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Small changes https://github.com/restic/rest-server/pull/11#issuecomment-309879710 https://github.com/restic/restic/issues/1055 https://github.com/restic/restic/pull/1077 + https://github.com/restic/restic/pull/1105 * When no S3 credentials are specified in the environment variables, restic now tries to load credentials from an IAM instance profile when the s3