From 47186806cfa0c6a96afe7e32756cd0e6e02a5cad Mon Sep 17 00:00:00 2001 From: Chris Howey Date: Sun, 10 May 2015 10:20:58 -0500 Subject: [PATCH] Add Amazon S3 backend. --- backend/s3/s3.go | 200 +++++++++++++++++++++++++++++++++++++++++++++ cmd/restic/main.go | 6 ++ 2 files changed, 206 insertions(+) create mode 100644 backend/s3/s3.go diff --git a/backend/s3/s3.go b/backend/s3/s3.go new file mode 100644 index 000000000..47ac21871 --- /dev/null +++ b/backend/s3/s3.go @@ -0,0 +1,200 @@ +package s3 + +import ( + "bytes" + "errors" + "io" + "strings" + "sync" + + "github.com/mitchellh/goamz/aws" + as3 "github.com/mitchellh/goamz/s3" + + "github.com/restic/restic/backend" +) + +var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") + +func s3path(t backend.Type, name string) string { + if t == backend.Config { + return string(t) + } + return string(t) + "/" + name +} + +type S3DB struct { + bucket *as3.Bucket + mput sync.Mutex + path string +} + +// Open opens the local backend at bucket and region. +func Open(regionname, bucketname string) (*S3DB, error) { + auth, err := aws.EnvAuth() + if err != nil { + return nil, err + } + + client := as3.New(auth, aws.Regions[regionname]) + + return &S3DB{bucket: client.Bucket(bucketname), path: bucketname}, nil +} + +// Location returns this backend's location (the directory name). +func (b *S3DB) Location() string { + return b.path +} + +type s3Blob struct { + b *S3DB + buf *bytes.Buffer + final bool +} + +func (bb *s3Blob) Write(p []byte) (int, error) { + if bb.final { + return 0, errors.New("blob already closed") + } + + n, err := bb.buf.Write(p) + return n, err +} + +func (bb *s3Blob) Read(p []byte) (int, error) { + return bb.buf.Read(p) +} + +func (bb *s3Blob) Close() error { + bb.final = true + bb.buf.Reset() + return nil +} + +func (bb *s3Blob) Size() uint { + return uint(bb.buf.Len()) +} + +func (bb *s3Blob) Finalize(t backend.Type, name string) error { + if bb.final { + return errors.New("Already finalized") + } + + bb.final = true + + path := s3path(t, name) + bb.b.mput.Lock() + err := bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private") + bb.b.mput.Unlock() + bb.buf.Reset() + return err +} + +// Create creates a new Blob. The data is available only after Finalize() +// has been called on the returned Blob. +func (b *S3DB) Create() (backend.Blob, error) { + blob := s3Blob{ + b: b, + buf: &bytes.Buffer{}, + } + + return &blob, nil +} + +func (b *S3DB) get(t backend.Type, name string) (*s3Blob, error) { + blob := &s3Blob{ + b: b, + } + + path := s3path(t, name) + data, err := b.bucket.Get(path) + blob.buf = bytes.NewBuffer(data) + return blob, err +} + +// Get returns a reader that yields the content stored under the given +// name. The reader should be closed after draining it. +func (b *S3DB) Get(t backend.Type, name string) (io.ReadCloser, error) { + return b.get(t, name) +} + +// GetReader returns an io.ReadCloser for the Blob with the given name of +// type t at offset and length. If length is 0, the reader reads until EOF. +func (b *S3DB) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { + blob, err := b.get(t, name) + if err != nil { + return nil, err + } + + blob.buf.Next(int(offset)) + + if length == 0 { + return blob, nil + } + + return backend.LimitReadCloser(blob, int64(length)), nil +} + +// Test returns true if a blob of the given type and name exists in the backend. +func (b *S3DB) Test(t backend.Type, name string) (bool, error) { + found := false + path := s3path(t, name) + key, err := b.bucket.GetKey(path) + if err == nil && key.Key == path { + found = true + } + + // If error, then not found + return found, nil +} + +// Remove removes the blob with the given name and type. +func (b *S3DB) Remove(t backend.Type, name string) error { + path := s3path(t, name) + return b.bucket.Del(path) +} + +// List returns a channel that yields all names of blobs of type t. A +// goroutine is started for this. If the channel done is closed, sending +// stops. +func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string { + ch := make(chan string) + + prefix := string(t) + "/" + + listresp, err := b.bucket.List(prefix, "/", "", 10000) + + if err != nil { + close(ch) + return ch + } + + matches := make([]string, len(listresp.Contents)) + for idx, key := range listresp.Contents { + matches[idx] = strings.TrimPrefix(key.Key, prefix) + } + + go func() { + defer close(ch) + for _, m := range matches { + if m == "" { + continue + } + + select { + case ch <- m: + case <-done: + return + } + } + }() + + return ch +} + +// Delete removes the repository and all files. +func (b *S3DB) Delete() error { + return b.bucket.DelBucket() +} + +// Close does nothing +func (b *S3DB) Close() error { return nil } diff --git a/cmd/restic/main.go b/cmd/restic/main.go index b34755c84..86baa34ab 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -12,6 +12,7 @@ import ( "github.com/jessevdk/go-flags" "github.com/restic/restic/backend" "github.com/restic/restic/backend/local" + "github.com/restic/restic/backend/s3" "github.com/restic/restic/backend/sftp" "github.com/restic/restic/debug" "github.com/restic/restic/repository" @@ -119,6 +120,7 @@ func (cmd CmdInit) Execute(args []string) error { // Open the backend specified by URI. // Valid formats are: // * /foo/bar -> local repository at /foo/bar +// * s3://region/bucket -> amazon s3 bucket // * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar // * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup func open(u string) (backend.Backend, error) { @@ -129,6 +131,8 @@ func open(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Open(url.Path) + } else if url.Scheme == "s3" { + return s3.Open(url.Host, url.Path[1:]) } args := []string{url.Host} @@ -150,6 +154,8 @@ func create(u string) (backend.Backend, error) { if url.Scheme == "" { return local.Create(url.Path) + } else if url.Scheme == "s3" { + return s3.Open(url.Host, url.Path[1:]) } args := []string{url.Host}