Merge pull request #918 from restic/fix-917

s3: Correct layout prefix, reduce connections

src/restic/backend/layout_s3.go

@@ -18,13 +18,29 @@ var s3LayoutPaths = map[restic.FileType]string{
 	restic.KeyFile:      "key",
 }
 
+// join calls Join with the first empty elements removed.
+func (l *S3Layout) join(url string, items ...string) string {
+	for len(items) > 0 && items[0] == "" {
+		items = items[1:]
+	}
+
+	path := l.Join(items...)
+	if path == "" || path[0] != '/' {
+		if url != "" && url[len(url)-1] != '/' {
+			url += "/"
+		}
+	}
+
+	return url + path
+}
+
 // Dirname returns the directory path for a given file type and name.
 func (l *S3Layout) Dirname(h restic.Handle) string {
 	if h.Type == restic.ConfigFile {
 		return l.URL + l.Join(l.Path, "/")
 	}
 
-	return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type]) + "/"
+	return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/"
 }
 
 // Filename returns a path to a file, including its name.
@@ -35,7 +51,7 @@ func (l *S3Layout) Filename(h restic.Handle) string {
 		name = "config"
 	}
 
-	return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type], name)
+	return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name)
 }
 
 // Paths returns all directory names

src/restic/backend/layout_test.go

@@ -190,6 +190,24 @@ func TestCloudLayoutURLs(t *testing.T) {
 			"https://hostname.foo:1234/prefix/repo/config",
 			"https://hostname.foo:1234/prefix/repo/",
 		},
+		{
+			&S3Layout{URL: "", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.DataFile, Name: "foobar"},
+			"data/foobar",
+			"data/",
+		},
+		{
+			&S3Layout{URL: "", Path: "", Join: path.Join},
+			restic.Handle{Type: restic.LockFile, Name: "foobar"},
+			"lock/foobar",
+			"lock/",
+		},
+		{
+			&S3Layout{URL: "", Path: "/", Join: path.Join},
+			restic.Handle{Type: restic.ConfigFile, Name: "foobar"},
+			"/config",
+			"/",
+		},
 	}
 
 	for _, test := range tests {

src/restic/backend/s3/s3.go

@@ -17,7 +17,7 @@ import (
 	"restic/debug"
 )
 
-const connLimit = 40
+const connLimit = 10
 
 // s3 is a backend which stores the data on an S3 endpoint.
 type s3 struct {
@@ -45,7 +45,7 @@ func Open(cfg Config) (restic.Backend, error) {
 		bucketname:   cfg.Bucket,
 		prefix:       cfg.Prefix,
 		cacheObjSize: make(map[string]int64),
-		Layout:       &backend.S3Layout{URL: cfg.Endpoint, Join: path.Join},
+		Layout:       &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
 	}
 
 	tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
@@ -88,10 +88,10 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) {
 		return err
 	}
 
-	debug.Log("Save %v", h)
-
 	objName := be.Filename(h)
 
+	debug.Log("Save %v at %v", h, objName)
+
 	// Check key does not already exist
 	_, err = be.client.StatObject(be.bucketname, objName)
 	if err == nil {
@@ -128,7 +128,7 @@ func (wr wrapReader) Close() error {
 // given offset. If length is nonzero, only a portion of the file is
 // returned. rd must be closed after use.
 func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
-	debug.Log("Load %v, length %v, offset %v", h, length, offset)
+	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
 	if err := h.Valid(); err != nil {
 		return nil, err
 	}
@@ -280,7 +280,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) {
 func (be *s3) Remove(h restic.Handle) error {
 	objName := be.Filename(h)
 	err := be.client.RemoveObject(be.bucketname, objName)
-	debug.Log("Remove(%v) -> err %v", h, err)
+	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
 	return errors.Wrap(err, "client.RemoveObject")
 }