1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2024-12-25 01:06:39 +00:00

Add Amazon S3 backend.

This commit is contained in:
Chris Howey 2015-05-10 10:20:58 -05:00
parent c37ae44ff5
commit 47186806cf
2 changed files with 206 additions and 0 deletions

200
backend/s3/s3.go Normal file
View file

@ -0,0 +1,200 @@
package s3
import (
"bytes"
"errors"
"io"
"strings"
"sync"
"github.com/mitchellh/goamz/aws"
as3 "github.com/mitchellh/goamz/s3"
"github.com/restic/restic/backend"
)
var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match")
func s3path(t backend.Type, name string) string {
if t == backend.Config {
return string(t)
}
return string(t) + "/" + name
}
type S3DB struct {
bucket *as3.Bucket
mput sync.Mutex
path string
}
// Open opens the local backend at bucket and region.
func Open(regionname, bucketname string) (*S3DB, error) {
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
client := as3.New(auth, aws.Regions[regionname])
return &S3DB{bucket: client.Bucket(bucketname), path: bucketname}, nil
}
// Location returns this backend's location (the directory name).
func (b *S3DB) Location() string {
return b.path
}
type s3Blob struct {
b *S3DB
buf *bytes.Buffer
final bool
}
func (bb *s3Blob) Write(p []byte) (int, error) {
if bb.final {
return 0, errors.New("blob already closed")
}
n, err := bb.buf.Write(p)
return n, err
}
func (bb *s3Blob) Read(p []byte) (int, error) {
return bb.buf.Read(p)
}
func (bb *s3Blob) Close() error {
bb.final = true
bb.buf.Reset()
return nil
}
func (bb *s3Blob) Size() uint {
return uint(bb.buf.Len())
}
func (bb *s3Blob) Finalize(t backend.Type, name string) error {
if bb.final {
return errors.New("Already finalized")
}
bb.final = true
path := s3path(t, name)
bb.b.mput.Lock()
err := bb.b.bucket.Put(path, bb.buf.Bytes(), "binary/octet-stream", "private")
bb.b.mput.Unlock()
bb.buf.Reset()
return err
}
// Create creates a new Blob. The data is available only after Finalize()
// has been called on the returned Blob.
func (b *S3DB) Create() (backend.Blob, error) {
blob := s3Blob{
b: b,
buf: &bytes.Buffer{},
}
return &blob, nil
}
func (b *S3DB) get(t backend.Type, name string) (*s3Blob, error) {
blob := &s3Blob{
b: b,
}
path := s3path(t, name)
data, err := b.bucket.Get(path)
blob.buf = bytes.NewBuffer(data)
return blob, err
}
// Get returns a reader that yields the content stored under the given
// name. The reader should be closed after draining it.
func (b *S3DB) Get(t backend.Type, name string) (io.ReadCloser, error) {
return b.get(t, name)
}
// GetReader returns an io.ReadCloser for the Blob with the given name of
// type t at offset and length. If length is 0, the reader reads until EOF.
func (b *S3DB) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) {
blob, err := b.get(t, name)
if err != nil {
return nil, err
}
blob.buf.Next(int(offset))
if length == 0 {
return blob, nil
}
return backend.LimitReadCloser(blob, int64(length)), nil
}
// Test returns true if a blob of the given type and name exists in the backend.
func (b *S3DB) Test(t backend.Type, name string) (bool, error) {
found := false
path := s3path(t, name)
key, err := b.bucket.GetKey(path)
if err == nil && key.Key == path {
found = true
}
// If error, then not found
return found, nil
}
// Remove removes the blob with the given name and type.
func (b *S3DB) Remove(t backend.Type, name string) error {
path := s3path(t, name)
return b.bucket.Del(path)
}
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *S3DB) List(t backend.Type, done <-chan struct{}) <-chan string {
ch := make(chan string)
prefix := string(t) + "/"
listresp, err := b.bucket.List(prefix, "/", "", 10000)
if err != nil {
close(ch)
return ch
}
matches := make([]string, len(listresp.Contents))
for idx, key := range listresp.Contents {
matches[idx] = strings.TrimPrefix(key.Key, prefix)
}
go func() {
defer close(ch)
for _, m := range matches {
if m == "" {
continue
}
select {
case ch <- m:
case <-done:
return
}
}
}()
return ch
}
// Delete removes the repository and all files.
func (b *S3DB) Delete() error {
return b.bucket.DelBucket()
}
// Close does nothing
func (b *S3DB) Close() error { return nil }

View file

@ -12,6 +12,7 @@ import (
"github.com/jessevdk/go-flags"
"github.com/restic/restic/backend"
"github.com/restic/restic/backend/local"
"github.com/restic/restic/backend/s3"
"github.com/restic/restic/backend/sftp"
"github.com/restic/restic/debug"
"github.com/restic/restic/repository"
@ -119,6 +120,7 @@ func (cmd CmdInit) Execute(args []string) error {
// Open the backend specified by URI.
// Valid formats are:
// * /foo/bar -> local repository at /foo/bar
// * s3://region/bucket -> amazon s3 bucket
// * sftp://user@host/foo/bar -> remote sftp repository on host for user at path foo/bar
// * sftp://host//tmp/backup -> remote sftp repository on host at path /tmp/backup
func open(u string) (backend.Backend, error) {
@ -129,6 +131,8 @@ func open(u string) (backend.Backend, error) {
if url.Scheme == "" {
return local.Open(url.Path)
} else if url.Scheme == "s3" {
return s3.Open(url.Host, url.Path[1:])
}
args := []string{url.Host}
@ -150,6 +154,8 @@ func create(u string) (backend.Backend, error) {
if url.Scheme == "" {
return local.Create(url.Path)
} else if url.Scheme == "s3" {
return s3.Open(url.Host, url.Path[1:])
}
args := []string{url.Host}