From 8f5ff379b7d79f21f116db6ba54d73b49ef97e7e Mon Sep 17 00:00:00 2001 From: Christian Kemper Date: Sun, 7 Feb 2016 11:28:29 -0800 Subject: [PATCH] Introduced a configurable object path prefix for s3 repositories. Prepends the object path prefix to all s3 paths and allows to have multiple independent restic backup repositories in a single s3 bucket. Removed the hardcoded "restic" prefix from s3 paths. Use "restic" as the default object path prefix for s3 if no other prefix gets specified. This will retain backward compatibility with existing s3 repository configurations. Simplified the parse flow to have a single point where we parse the bucket name and the prefix within the bucket. Added tests for s3 object path prefix and the new default prefix to config_test and location_test. --- backend/s3/config.go | 76 +++++++++++++++++---------------------- backend/s3/config_test.go | 20 +++++++++++ backend/s3/s3.go | 22 ++++++------ location/location_test.go | 26 ++++++++++++++ 4 files changed, 90 insertions(+), 54 deletions(-) diff --git a/backend/s3/config.go b/backend/s3/config.go index b0224925a..4dab8a25c 100644 --- a/backend/s3/config.go +++ b/backend/s3/config.go @@ -13,53 +13,28 @@ type Config struct { UseHTTP bool KeyID, Secret string Bucket string + Prefix string } +const defaultPrefix = "restic" + // ParseConfig parses the string s and extracts the s3 config. The two -// supported configuration formats are s3://host/bucketname and -// s3:host:bucketname. The host can also be a valid s3 region name. +// supported configuration formats are s3://host/bucketname/prefix and +// s3:host:bucketname/prefix. The host can also be a valid s3 region +// name. If no prefix is given the prefix "restic" will be used. func ParseConfig(s string) (interface{}, error) { + var path []string + cfg := Config{} if strings.HasPrefix(s, "s3://") { s = s[5:] - - data := strings.SplitN(s, "/", 2) - if len(data) != 2 { - return nil, errors.New("s3: invalid format, host/region or bucket name not found") - } - - cfg := Config{ - Endpoint: data[0], - Bucket: data[1], - } - - return cfg, nil - } - - data := strings.SplitN(s, ":", 2) - if len(data) != 2 { - return nil, errors.New("s3: invalid format") - } - - if data[0] != "s3" { - return nil, errors.New(`s3: config does not start with "s3"`) - } - - s = data[1] - - cfg := Config{} - rest := strings.Split(s, "/") - if len(rest) < 2 { - return nil, errors.New("s3: region or bucket not found") - } - - if len(rest) == 2 { - // assume that just a region name and a bucket has been specified, in - // the format region/bucket - cfg.Endpoint = rest[0] - cfg.Bucket = rest[1] - } else { - // assume that a URL has been specified, parse it and use the path as - // the bucket name. + path = strings.SplitN(s, "/", 3) + cfg.Endpoint = path[0] + path = path[1:] + } else if strings.HasPrefix(s, "s3:http") { + s = s[3:] + // assume that a URL has been specified, parse it and + // use the host as the endpoint and the path as the + // bucket name and prefix url, err := url.Parse(s) if err != nil { return nil, err @@ -73,8 +48,23 @@ func ParseConfig(s string) (interface{}, error) { if url.Scheme == "http" { cfg.UseHTTP = true } - - cfg.Bucket = url.Path[1:] + path = strings.SplitN(url.Path[1:], "/", 2) + } else if strings.HasPrefix(s, "s3:") { + s = s[3:] + path = strings.SplitN(s, "/", 3) + cfg.Endpoint = path[0] + path = path[1:] + } else { + return nil, errors.New("s3: invalid format") + } + if len(path) < 1 { + return nil, errors.New("s3: invalid format, host/region or bucket name not found") + } + cfg.Bucket = path[0] + if len(path) > 1 { + cfg.Prefix = path[1] + } else { + cfg.Prefix = defaultPrefix } return cfg, nil diff --git a/backend/s3/config_test.go b/backend/s3/config_test.go index 54fc4718a..d1ecbdf60 100644 --- a/backend/s3/config_test.go +++ b/backend/s3/config_test.go @@ -9,18 +9,38 @@ var configTests = []struct { {"s3://eu-central-1/bucketname", Config{ Endpoint: "eu-central-1", Bucket: "bucketname", + Prefix: "restic", + }}, + {"s3://eu-central-1/bucketname/prefix/directory", Config{ + Endpoint: "eu-central-1", + Bucket: "bucketname", + Prefix: "prefix/directory", }}, {"s3:eu-central-1/foobar", Config{ Endpoint: "eu-central-1", Bucket: "foobar", + Prefix: "restic", + }}, + {"s3:eu-central-1/foobar/prefix/directory", Config{ + Endpoint: "eu-central-1", + Bucket: "foobar", + Prefix: "prefix/directory", }}, {"s3:https://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", + Prefix: "restic", }}, {"s3:http://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", + Prefix: "restic", + UseHTTP: true, + }}, + {"s3:http://hostname:9999/bucket/prefix/directory", Config{ + Endpoint: "hostname:9999", + Bucket: "bucket", + Prefix: "prefix/directory", UseHTTP: true, }}, } diff --git a/backend/s3/s3.go b/backend/s3/s3.go index ccb6cd42c..0bdf47894 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -13,13 +13,12 @@ import ( ) const connLimit = 10 -const backendPrefix = "restic" -func s3path(t backend.Type, name string) string { +func s3path(prefix string, t backend.Type, name string) string { if t == backend.Config { - return backendPrefix + "/" + string(t) + return prefix + "/" + string(t) } - return backendPrefix + "/" + string(t) + "/" + name + return prefix + "/" + string(t) + "/" + name } // s3 is a backend which stores the data on an S3 endpoint. @@ -27,6 +26,7 @@ type s3 struct { client minio.CloudStorageClient connChan chan struct{} bucketname string + prefix string } // Open opens the S3 backend at bucket and region. The bucket is created if it @@ -39,7 +39,7 @@ func Open(cfg Config) (backend.Backend, error) { return nil, err } - be := &s3{client: client, bucketname: cfg.Bucket} + be := &s3{client: client, bucketname: cfg.Bucket, prefix: cfg.Prefix} be.createConnections() if err := client.BucketExists(cfg.Bucket); err != nil { @@ -72,7 +72,7 @@ func (be *s3) Location() string { // and saves it in p. Load has the same semantics as io.ReaderAt. func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) { debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p)) - path := s3path(h.Type, h.Name) + path := s3path(be.prefix, h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) if err != nil { debug.Log("s3.GetReader", " err %v", err) @@ -101,7 +101,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) { debug.Log("s3.Save", "%v bytes at %d", len(p), h) - path := s3path(h.Type, h.Name) + path := s3path(be.prefix, h.Type, h.Name) // Check key does not already exist _, err = be.client.StatObject(be.bucketname, path) @@ -126,7 +126,7 @@ func (be s3) Save(h backend.Handle, p []byte) (err error) { // Stat returns information about a blob. func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) { debug.Log("s3.Stat", "%v") - path := s3path(h.Type, h.Name) + path := s3path(be.prefix, h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) if err != nil { debug.Log("s3.Stat", "GetObject() err %v", err) @@ -145,7 +145,7 @@ func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) { // Test returns true if a blob of the given type and name exists in the backend. func (be *s3) Test(t backend.Type, name string) (bool, error) { found := false - path := s3path(t, name) + path := s3path(be.prefix, t, name) _, err := be.client.StatObject(be.bucketname, path) if err == nil { found = true @@ -157,7 +157,7 @@ func (be *s3) Test(t backend.Type, name string) (bool, error) { // Remove removes the blob with the given name and type. func (be *s3) Remove(t backend.Type, name string) error { - path := s3path(t, name) + path := s3path(be.prefix, t, name) err := be.client.RemoveObject(be.bucketname, path) debug.Log("s3.Remove", "%v %v -> err %v", t, name, err) return err @@ -170,7 +170,7 @@ func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string { debug.Log("s3.List", "listing %v", t) ch := make(chan string) - prefix := s3path(t, "") + prefix := s3path(be.prefix, t, "") listresp := be.client.ListObjects(be.bucketname, prefix, true, done) diff --git a/location/location_test.go b/location/location_test.go index b0303fad1..ef827dcdd 100644 --- a/location/location_test.go +++ b/location/location_test.go @@ -48,30 +48,56 @@ var parseTests = []struct { Config: s3.Config{ Endpoint: "eu-central-1", Bucket: "bucketname", + Prefix: "restic", }}, }, {"s3://hostname.foo/bucketname", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "bucketname", + Prefix: "restic", + }}, + }, + {"s3://hostname.foo/bucketname/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "bucketname", + Prefix: "prefix/directory", }}, }, {"s3:eu-central-1/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "eu-central-1", Bucket: "repo", + Prefix: "restic", + }}, + }, + {"s3:eu-central-1/repo/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "eu-central-1", + Bucket: "repo", + Prefix: "prefix/directory", }}, }, {"s3:https://hostname.foo/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", + Prefix: "restic", + }}, + }, + {"s3:https://hostname.foo/repo/prefix/directory", Location{Scheme: "s3", + Config: s3.Config{ + Endpoint: "hostname.foo", + Bucket: "repo", + Prefix: "prefix/directory", }}, }, {"s3:http://hostname.foo/repo", Location{Scheme: "s3", Config: s3.Config{ Endpoint: "hostname.foo", Bucket: "repo", + Prefix: "restic", UseHTTP: true, }}, },