From 35636a9d929200e0deb7b0d47396895895cc5264 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 15 Feb 2015 17:09:49 +0100 Subject: [PATCH] Introduce CreateBlob() method for backend --- backend/interface.go | 6 ++++ backend/local.go | 74 +++++++++++++++++++++++++++++++++++++++++ backend/local_test.go | 9 ++++- backend/sftp.go | 76 +++++++++++++++++++++++++++++++++++++++++++ server.go | 4 +++ 5 files changed, 168 insertions(+), 1 deletion(-) diff --git a/backend/interface.go b/backend/interface.go index 8eeb7bfe2..3836daa7b 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -23,6 +23,11 @@ var ( ErrAlreadyPresent = errors.New("blob is already present in backend") ) +type Blob interface { + io.WriteCloser + ID() (ID, error) +} + type Lister interface { List(Type) (IDs, error) } @@ -35,6 +40,7 @@ type Getter interface { type Creater interface { Create(Type, []byte) (ID, error) CreateFrom(Type, io.Reader) (ID, error) + CreateBlob(Type) (Blob, error) } type Tester interface { diff --git a/backend/local.go b/backend/local.go index 6b7402287..338149de6 100644 --- a/backend/local.go +++ b/backend/local.go @@ -3,6 +3,7 @@ package backend import ( "errors" "fmt" + "hash" "io" "io/ioutil" "os" @@ -272,6 +273,79 @@ func (b *Local) CreateFrom(t Type, rd io.Reader) (ID, error) { return id, nil } +type localBlob struct { + f *os.File + h hash.Hash + tw io.Writer + backend *Local + tpe Type + id ID +} + +func (lb *localBlob) Close() error { + err := lb.f.Close() + if err != nil { + return err + } + + // get ID + lb.id = ID(lb.h.Sum(nil)) + + // check for duplicate ID + res, err := lb.backend.Test(lb.tpe, lb.id) + if err != nil { + return fmt.Errorf("testing presence of ID %v failed: %v", lb.id, err) + } + + if res { + return ErrAlreadyPresent + } + + // rename file + err = lb.backend.renameFile(lb.f, lb.tpe, lb.id) + if err != nil { + return err + } + + return nil +} + +func (lb *localBlob) Write(p []byte) (int, error) { + return lb.tw.Write(p) +} + +func (lb *localBlob) ID() (ID, error) { + if lb.id == nil { + return nil, errors.New("blob is not closed, ID unavailable") + } + + return lb.id, nil +} + +// Create creates a new blob of type t. Blob implements io.WriteCloser. Once +// Close() has been called, ID() can be used to retrieve the ID. If the blob is +// already present, Close() returns ErrAlreadyPresent. +func (b *Local) CreateBlob(t Type) (Blob, error) { + // TODO: make sure that tempfile is removed upon error + + // create tempfile in backend + file, err := b.tempFile() + if err != nil { + return nil, err + } + + h := newHash() + blob := localBlob{ + h: h, + tw: io.MultiWriter(h, file), + f: file, + backend: b, + tpe: t, + } + + return &blob, nil +} + // Construct path for given Type and ID. func (b *Local) filename(t Type, id ID) string { return filepath.Join(b.dirname(t, id), id.String()) diff --git a/backend/local_test.go b/backend/local_test.go index 7e085e62d..1179f1462 100644 --- a/backend/local_test.go +++ b/backend/local_test.go @@ -69,7 +69,14 @@ func testBackend(b backend.Backend, t *testing.T) { // add files for _, test := range TestStrings { // store string in backend - id, err := b.Create(tpe, []byte(test.data)) + blob, err := b.CreateBlob(tpe) + ok(t, err) + + _, err = blob.Write([]byte(test.data)) + ok(t, err) + ok(t, blob.Close()) + + id, err := blob.ID() ok(t, err) equals(t, test.id, id.String()) diff --git a/backend/sftp.go b/backend/sftp.go index 92e604808..2c73ae393 100644 --- a/backend/sftp.go +++ b/backend/sftp.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "errors" "fmt" + "hash" "io" "io/ioutil" "log" @@ -380,6 +381,81 @@ func (r *SFTP) CreateFrom(t Type, rd io.Reader) (ID, error) { return id, nil } +type sftpBlob struct { + f *sftp.File + name string + h hash.Hash + tw io.Writer + backend *SFTP + tpe Type + id ID +} + +func (sb *sftpBlob) Close() error { + err := sb.f.Close() + if err != nil { + return err + } + + // get ID + sb.id = ID(sb.h.Sum(nil)) + + // check for duplicate ID + res, err := sb.backend.Test(sb.tpe, sb.id) + if err != nil { + return fmt.Errorf("testing presence of ID %v failed: %v", sb.id, err) + } + + if res { + return ErrAlreadyPresent + } + + // rename file + err = sb.backend.renameFile(sb.name, sb.tpe, sb.id) + if err != nil { + return err + } + + return nil +} + +func (sb *sftpBlob) Write(p []byte) (int, error) { + return sb.tw.Write(p) +} + +func (sb *sftpBlob) ID() (ID, error) { + if sb.id == nil { + return nil, errors.New("blob is not closed, ID unavailable") + } + + return sb.id, nil +} + +// Create creates a new blob of type t. Blob implements io.WriteCloser. Once +// Close() has been called, ID() can be used to retrieve the ID. If the blob is +// already present, Close() returns ErrAlreadyPresent. +func (r *SFTP) CreateBlob(t Type) (Blob, error) { + // TODO: make sure that tempfile is removed upon error + + // create tempfile in backend + filename, file, err := r.tempFile() + if err != nil { + return nil, err + } + + h := newHash() + blob := sftpBlob{ + h: h, + tw: io.MultiWriter(h, file), + f: file, + name: filename, + backend: r, + tpe: t, + } + + return &blob, nil +} + // Construct path for given Type and ID. func (r *SFTP) filename(t Type, id ID) string { return filepath.Join(r.dirname(t, id), id.String()) diff --git a/server.go b/server.go index 425c9d546..e09e9ff5c 100644 --- a/server.go +++ b/server.go @@ -330,6 +330,10 @@ func (s Server) CreateFrom(t backend.Type, r io.Reader) (backend.ID, error) { return s.be.CreateFrom(t, r) } +func (s Server) CreateBlob(t backend.Type) (backend.Blob, error) { + return s.be.CreateBlob(t) +} + func (s Server) Test(t backend.Type, id backend.ID) (bool, error) { return s.be.Test(t, id) }