restic/server.go

419 lines
9.2 KiB
Go
Raw Normal View History

2014-12-21 16:02:49 +00:00
package restic
import (
2015-02-15 23:24:58 +00:00
"crypto/sha256"
"encoding/json"
2014-12-21 16:02:49 +00:00
"errors"
"fmt"
"io"
2015-03-28 10:50:23 +00:00
"io/ioutil"
"sync"
2014-12-21 16:02:49 +00:00
"github.com/restic/restic/backend"
"github.com/restic/restic/chunker"
"github.com/restic/restic/crypto"
"github.com/restic/restic/debug"
2014-12-21 16:02:49 +00:00
)
type Server struct {
be backend.Backend
key *Key
}
func NewServer(be backend.Backend) Server {
return Server{be: be}
}
func (s *Server) SetKey(k *Key) {
s.key = k
2014-12-21 16:02:49 +00:00
}
// ChunkerPolynomial returns the secret polynomial used for content defined chunking.
func (s *Server) ChunkerPolynomial() chunker.Pol {
return chunker.Pol(s.key.Master().ChunkerPolynomial)
}
2015-03-28 10:50:23 +00:00
// Find loads the list of all blobs of type t and searches for names which start
2014-12-21 16:02:49 +00:00
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
// more than one is found, nil and ErrMultipleIDMatches is returned.
2015-03-28 10:50:23 +00:00
func (s Server) Find(t backend.Type, prefix string) (string, error) {
2014-12-21 16:02:49 +00:00
return backend.Find(s.be, t, prefix)
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
2015-03-28 10:50:23 +00:00
func (s Server) FindSnapshot(name string) (string, error) {
return backend.FindSnapshot(s.be, name)
2014-12-21 16:02:49 +00:00
}
// PrefixLength returns the number of bytes required so that all prefixes of
// all IDs of type t are unique.
func (s Server) PrefixLength(t backend.Type) (int, error) {
return backend.PrefixLength(s.be, t)
}
// Load tries to load and decrypt content identified by t and blob from the
// backend. If the blob specifies an ID, the decrypted plaintext is checked
// against this ID. The same goes for blob.Size and blob.StorageSize: If they
// are set to a value > 0, this value is checked.
func (s Server) Load(t backend.Type, blob Blob) ([]byte, error) {
// load data
rd, err := s.be.Get(t, blob.Storage.String())
2015-03-28 10:50:23 +00:00
if err != nil {
return nil, err
}
buf, err := ioutil.ReadAll(rd)
if err != nil {
return nil, err
}
2015-03-28 10:50:23 +00:00
// check hash
if !backend.Hash(buf).Equal(blob.Storage) {
return nil, errors.New("invalid data returned")
}
// check length
if blob.StorageSize > 0 && len(buf) != int(blob.StorageSize) {
return nil, errors.New("Invalid storage length")
}
// decrypt
buf, err = s.Decrypt(buf)
if err != nil {
return nil, err
}
// check length
if blob.Size > 0 && len(buf) != int(blob.Size) {
return nil, errors.New("Invalid length")
}
// check SHA256 sum
if blob.ID != nil {
id := backend.Hash(buf)
if !blob.ID.Equal(id) {
return nil, fmt.Errorf("load %v: expected plaintext hash %v, got %v", blob.Storage, blob.ID, id)
}
}
return buf, nil
}
// Load tries to load and decrypt content identified by t and id from the backend.
func (s Server) LoadID(t backend.Type, storageID backend.ID) ([]byte, error) {
return s.Load(t, Blob{Storage: storageID})
}
// LoadJSON calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (s Server) LoadJSON(t backend.Type, blob Blob, item interface{}) error {
buf, err := s.Load(t, blob)
if err != nil {
return err
}
return json.Unmarshal(buf, item)
}
// LoadJSONID calls Load() to get content from the backend and afterwards calls
// json.Unmarshal on the item.
func (s Server) LoadJSONID(t backend.Type, id backend.ID, item interface{}) error {
2015-02-15 22:48:59 +00:00
// read
rd, err := s.be.Get(t, id.String())
2015-02-15 22:48:59 +00:00
if err != nil {
return err
}
defer rd.Close()
2015-02-15 22:48:59 +00:00
// decrypt
decryptRd, err := s.key.DecryptFrom(rd)
2015-02-17 22:37:45 +00:00
defer decryptRd.Close()
2015-02-15 22:48:59 +00:00
if err != nil {
return err
}
// decode
2015-03-02 08:56:56 +00:00
decoder := json.NewDecoder(decryptRd)
2015-02-15 22:48:59 +00:00
err = decoder.Decode(item)
if err != nil {
return err
}
2015-02-15 22:48:59 +00:00
return nil
}
// Save encrypts data and stores it to the backend as type t.
func (s Server) Save(t backend.Type, data []byte, id backend.ID) (Blob, error) {
if id == nil {
// compute plaintext hash
id = backend.Hash(data)
}
// create a new blob
blob := Blob{
ID: id,
Size: uint64(len(data)),
}
var ciphertext []byte
// if the data is small enough, use a slice from the pool
2015-04-12 07:41:47 +00:00
if len(data) <= maxCiphertextSize-crypto.Extension {
ciphertext = GetChunkBuf("ch.Save()")
defer FreeChunkBuf("ch.Save()", ciphertext)
} else {
2015-04-12 07:41:47 +00:00
l := len(data) + crypto.Extension
debug.Log("Server.Save", "create large slice of %d bytes for ciphertext", l)
// use a new slice
ciphertext = make([]byte, l)
}
// encrypt blob
ciphertext, err := s.Encrypt(ciphertext, data)
if err != nil {
return Blob{}, err
}
2015-03-28 10:50:23 +00:00
// compute ciphertext hash
sid := backend.Hash(ciphertext)
// save blob
backendBlob, err := s.be.Create()
if err != nil {
return Blob{}, err
}
_, err = backendBlob.Write(ciphertext)
if err != nil {
return Blob{}, err
}
2015-03-28 10:50:23 +00:00
err = backendBlob.Finalize(t, sid.String())
if err != nil {
return Blob{}, err
}
blob.Storage = sid
blob.StorageSize = uint64(len(ciphertext))
return blob, nil
}
// SaveFrom encrypts data read from rd and stores it to the backend as type t.
func (s Server) SaveFrom(t backend.Type, id backend.ID, length uint, rd io.Reader) (Blob, error) {
if id == nil {
return Blob{}, errors.New("id is nil")
}
backendBlob, err := s.be.Create()
if err != nil {
return Blob{}, err
}
2015-03-28 10:50:23 +00:00
hw := backend.NewHashingWriter(backendBlob, sha256.New())
encWr := s.key.EncryptTo(hw)
_, err = io.Copy(encWr, rd)
if err != nil {
return Blob{}, err
}
// finish encryption
err = encWr.Close()
if err != nil {
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
}
// finish backend blob
2015-03-28 10:50:23 +00:00
sid := backend.ID(hw.Sum(nil))
err = backendBlob.Finalize(t, sid.String())
if err != nil {
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
}
return Blob{
ID: id,
Size: uint64(length),
2015-03-28 10:50:23 +00:00
Storage: sid,
StorageSize: uint64(backendBlob.Size()),
}, nil
}
2015-02-15 23:24:58 +00:00
// SaveJSON serialises item as JSON and encrypts and saves it in the backend as
// type t.
func (s Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) {
backendBlob, err := s.be.Create()
if err != nil {
2015-02-15 23:24:58 +00:00
return Blob{}, fmt.Errorf("Create: %v", err)
}
2015-03-28 10:50:23 +00:00
storagehw := backend.NewHashingWriter(backendBlob, sha256.New())
encWr := s.key.EncryptTo(storagehw)
plainhw := backend.NewHashingWriter(encWr, sha256.New())
2015-02-15 23:24:58 +00:00
2015-03-28 10:50:23 +00:00
enc := json.NewEncoder(plainhw)
2015-02-15 23:24:58 +00:00
err = enc.Encode(item)
if err != nil {
return Blob{}, fmt.Errorf("json.NewEncoder: %v", err)
}
// finish encryption
err = encWr.Close()
if err != nil {
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
}
// finish backend blob
2015-03-28 10:50:23 +00:00
sid := backend.ID(storagehw.Sum(nil))
err = backendBlob.Finalize(t, sid.String())
2015-02-15 23:24:58 +00:00
if err != nil {
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
}
2015-03-28 10:50:23 +00:00
id := backend.ID(plainhw.Sum(nil))
2015-02-15 23:24:58 +00:00
return Blob{
ID: id,
2015-03-28 10:50:23 +00:00
Size: uint64(plainhw.Size()),
Storage: sid,
2015-02-15 23:24:58 +00:00
StorageSize: uint64(backendBlob.Size()),
}, nil
}
2014-12-21 16:02:49 +00:00
// Returns the backend used for this server.
func (s Server) Backend() backend.Backend {
return s.be
}
2014-12-21 17:10:19 +00:00
func (s *Server) SearchKey(password string) error {
key, err := SearchKey(*s, password)
if err != nil {
return err
}
s.key = key
return nil
}
func (s Server) Decrypt(ciphertext []byte) ([]byte, error) {
if s.key == nil {
return nil, errors.New("key for server not set")
}
Add 'plaintext' parameter to key.Decrypt() % benchcmp /tmp/{before,after} benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 261144414 260252046 -0.34% BenchmarkChunkEncryptParallel 261201438 261267029 +0.03% BenchmarkArchiveDirectory 0.00 0.00 +0.00% BenchmarkEncryptWriter 88297245 87526529 -0.87% BenchmarkEncrypt 87406446 87917897 +0.59% BenchmarkDecryptReader 89948630 90042541 +0.10% BenchmarkEncryptDecryptReader 178374144 179477306 +0.62% BenchmarkDecrypt 88289705 87937073 -0.40% BenchmarkSaveJSON 213917 213571 -0.16% BenchmarkSaveFrom 74881361 75111148 +0.31% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 40.15 40.29 1.00x BenchmarkChunkEncryptParallel 40.14 40.13 1.00x BenchmarkEncryptWriter 95.00 95.84 1.01x BenchmarkEncrypt 95.97 95.41 0.99x BenchmarkDecryptReader 93.26 93.16 1.00x BenchmarkEncryptDecryptReader 47.03 46.74 0.99x BenchmarkDecrypt 95.01 95.39 1.00x BenchmarkSaveFrom 56.01 55.84 1.00x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 113 112 -0.88% BenchmarkChunkEncryptParallel 104 103 -0.96% BenchmarkArchiveDirectory 0 0 +0.00% BenchmarkEncryptWriter 20 20 +0.00% BenchmarkEncrypt 14 14 +0.00% BenchmarkDecryptReader 18 18 +0.00% BenchmarkEncryptDecryptReader 40 45 +12.50% BenchmarkDecrypt 17 16 -5.88% BenchmarkSaveJSON 125 125 +0.00% BenchmarkSaveFrom 116 119 +2.59% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 8515750 118956 -98.60% BenchmarkChunkEncryptParallel 8515766 118972 -98.60% BenchmarkArchiveDirectory 0 0 +0.00% BenchmarkEncryptWriter 28927 28927 +0.00% BenchmarkEncrypt 422313 2473 -99.41% BenchmarkDecryptReader 527827 527827 +0.00% BenchmarkEncryptDecryptReader 4100824 4100870 +0.00% BenchmarkDecrypt 8391127 2509 -99.97% BenchmarkSaveJSON 9208 9208 +0.00% BenchmarkSaveFrom 39694 40541 +2.13%
2015-02-17 20:09:54 +00:00
return s.key.Decrypt([]byte{}, ciphertext)
2014-12-21 17:10:19 +00:00
}
func (s Server) Encrypt(ciphertext, plaintext []byte) ([]byte, error) {
2014-12-21 17:10:19 +00:00
if s.key == nil {
return nil, errors.New("key for server not set")
2014-12-21 17:10:19 +00:00
}
return s.key.Encrypt(ciphertext, plaintext)
}
func (s Server) Key() *Key {
return s.key
}
2015-02-17 22:05:23 +00:00
type ServerStats struct {
Blobs, Trees uint
}
2014-12-21 16:02:49 +00:00
2015-02-17 22:05:23 +00:00
// Stats returns statistics for this backend and the server.
func (s Server) Stats() (ServerStats, error) {
blobs := backend.NewIDSet()
// load all trees, in parallel
worker := func(wg *sync.WaitGroup, b <-chan Blob) {
for blob := range b {
tree, err := LoadTree(s, blob)
2015-02-17 22:05:23 +00:00
// ignore error and advance to next tree
if err != nil {
return
}
for _, id := range tree.Map.StorageIDs() {
blobs.Insert(id)
}
2014-12-21 16:02:49 +00:00
}
2015-02-17 22:05:23 +00:00
wg.Done()
}
2014-12-21 16:02:49 +00:00
blobCh := make(chan Blob)
2015-02-17 22:05:23 +00:00
// start workers
var wg sync.WaitGroup
for i := 0; i < maxConcurrency; i++ {
wg.Add(1)
go worker(&wg, blobCh)
2015-02-17 22:05:23 +00:00
}
2014-12-21 16:02:49 +00:00
2015-02-17 22:05:23 +00:00
// list ids
trees := 0
2015-03-28 10:50:23 +00:00
done := make(chan struct{})
defer close(done)
for name := range s.List(backend.Tree, done) {
2015-02-17 22:05:23 +00:00
trees++
2015-03-28 10:50:23 +00:00
id, err := backend.ParseID(name)
if err != nil {
debug.Log("Server.Stats", "unable to parse name %v as id: %v", name, err)
continue
}
blobCh <- Blob{Storage: id}
2015-03-28 10:50:23 +00:00
}
2015-02-17 22:05:23 +00:00
close(blobCh)
2015-02-17 22:05:23 +00:00
// wait for workers
wg.Wait()
2015-03-28 10:50:23 +00:00
return ServerStats{Blobs: uint(blobs.Len()), Trees: uint(trees)}, nil
2014-12-21 16:02:49 +00:00
}
2015-03-28 10:50:23 +00:00
// Count returns the number of blobs of a given type in the backend.
func (s Server) Count(t backend.Type) (n int) {
2015-03-28 14:52:15 +00:00
for _ = range s.List(t, nil) {
2015-03-28 10:50:23 +00:00
n++
2015-02-21 14:32:48 +00:00
}
2015-03-28 10:50:23 +00:00
return
2015-02-21 14:32:48 +00:00
}
2014-12-21 16:02:49 +00:00
// Proxy methods to backend
2015-03-28 10:50:23 +00:00
func (s Server) List(t backend.Type, done <-chan struct{}) <-chan string {
return s.be.List(t, done)
2014-12-21 16:02:49 +00:00
}
2015-03-28 10:50:23 +00:00
func (s Server) Test(t backend.Type, name string) (bool, error) {
return s.be.Test(t, name)
}
2015-03-28 10:50:23 +00:00
func (s Server) Remove(t backend.Type, name string) error {
return s.be.Remove(t, name)
2014-12-21 16:02:49 +00:00
}
func (s Server) Close() error {
return s.be.Close()
}
func (s Server) Delete() error {
if b, ok := s.be.(backend.Deleter); ok {
return b.Delete()
}
return errors.New("Delete() called for backend that does not implement this method")
}
2015-03-28 10:50:23 +00:00
func (s Server) ID() string {
return s.be.ID()
}
2015-03-28 10:50:23 +00:00
func (s Server) Location() string {
return s.be.Location()
}