From 062c328f2d31c03fab25a6af2029b64ef1ef95b7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 15:00:26 +0200 Subject: [PATCH 01/17] doc: Add config --- doc/Design.md | 113 ++++++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/doc/Design.md b/doc/Design.md index b815efad9..4463fadb3 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -21,49 +21,65 @@ been backed up at some point in time. The state here means the content and meta data like the name and modification time for the file or the directory and its contents. +*Storage ID*: A storage ID is the hash of the content of a file stored in the +repository. This ID is needed in order to load the file from the repository. +The storage hash is the SHA-256 hash of the content. + Repository Format ================= All data is stored in a restic repository. A repository is able to store data -of several different types, which can later be requested based on an ID. The ID -is the hash (SHA-256) of the content of a file. All files in a repository are -only written once and never modified afterwards. This allows accessing and even -writing to the repository with multiple clients in parallel. Only the delete -operation changes data in the repository. +of several different types, which can later be requested based on an ID. This +so-called "storage ID" is the hash (SHA-256) of the content of a file. All +files in a repository are only written once and never modified afterwards. This +allows accessing and even writing to the repository with multiple clients in +parallel. Only the delete operation removes data from the repository. At the time of writing, the only implemented repository type is based on directories and files. Such repositories can be accessed locally on the same system or via the integrated SFTP client. The directory layout is the same for both access methods. This repository type is described in the following. -Repositories consists of several directories and a file called `version`. This -file contains the version number of the repository. At the moment, this file -is expected to hold the string `1`, with an optional newline character. -Additionally there is a file named `id` which contains 32 random bytes, encoded -in hexadecimal. This uniquely identifies the repository, regardless if it is -accessed via SFTP or locally. +Repositories consists of several directories and a file called `config`. For +all other files stored in the repository, the name for the file is the lower +case hexadecimal representation of the storage ID, which is the SHA-256 hash of +the file's contents. This allows easily checking all files for accidental +modifications like disk read errors by simply running the program `sha256sum` +and comparing its output to the file name. If the prefix of a filename is +unique amongst all the other files in the same directory, the prefix may be +used instead of the complete filename. -For all other files stored in the repository, the name for the file is the -lower case hexadecimal representation of the SHA-256 hash of the file's -contents. This allows easily checking all files for accidental modifications -like disk read errors by simply running the program `sha256sum` and comparing -its output to the file name. If the prefix of a filename is unique amongst all -the other files in the same directory, the prefix may be used instead of the -complete filename. - -Apart from the files `version`, `id` and the files stored below the `keys` -directory, all files are encrypted with AES-256 in counter mode (CTR). The -integrity of the encrypted data is secured by a Poly1305-AES message -authentication code (sometimes also referred to as a "signature"). +Apart from the files stored below the `keys` directory, all files are encrypted +with AES-256 in counter mode (CTR). The integrity of the encrypted data is +secured by a Poly1305-AES message authentication code (sometimes also referred +to as a "signature"). In the first 16 bytes of each encrypted file the initialisation vector (IV) is stored. It is followed by the encrypted data and completed by the 16 byte MAC. The format is: `IV || CIPHERTEXT || MAC`. The complete encryption -overhead is 32 byte. For each file, a new random IV is selected. +overhead is 32 bytes. For each file, a new random IV is selected. -The basic layout of a sample restic repository is shown below: +The file `config` is encrypted this way and contains a JSON document like the +following: + + { + "version": 1, + "id": "5956a3f67a6230d4a92cefb29529f10196c7d92582ec305fd71ff6d331d6271b", + "chunker_polynomial": "25b468838dcb75" + } + +After decryption, restic first checks that the version field contains a version +number that it understands, otherwise it aborts. At the moment, the version is +expected to be 1. The field `id` holds a unique ID which consists of 32 +random bytes, encoded in hexadecimal. This uniquely identifies the repository, +regardless if it is accessed via SFTP or locally. The field +`chunker_polynomial` contains a parameter that is used for splitting large +files into smaller chunks (see below). + +The basic layout of a sample restic repository is shown here: /tmp/restic-repo + ├── config ├── data │ ├── 21 │ │ └── 2159dd48f8a24f33c307b750592773f8b71ff8d11452132a7b2e2a6a01611be1 @@ -74,7 +90,6 @@ The basic layout of a sample restic repository is shown below: │ ├── 73 │ │ └── 73d04e6125cf3c28a299cc2f3cca3b78ceac396e4fcf9575e34536b26782413c │ [...] - ├── id ├── index │ ├── c38f5fb68307c6a3e3aa945d556e325dc38f5fb68307c6a3e3aa945d556e325d │ └── ca171b1b7394d90d330b265d90f506f9984043b342525f019788f97e745c71fd @@ -83,8 +98,7 @@ The basic layout of a sample restic repository is shown below: ├── locks ├── snapshots │ └── 22a5af1bdc6e616f8a29579458c49627e01b32210d09adb288d1ecda7c5711ec - ├── tmp - └── version + └── tmp A repository can be initialized with the `restic init` command, e.g.: @@ -93,21 +107,21 @@ A repository can be initialized with the `restic init` command, e.g.: Pack Format ----------- -All files in the repository except Key and Data files just contain raw data, -stored as `IV || Ciphertext || MAC`. Data files may contain one or more Blobs -of data. The format is described in the following. +All files in the repository except Key and Pack files just contain raw data, +stored as `IV || Ciphertext || MAC`. Pack files may contain one or more Blobs +of data. -The Pack's structure is as follows: +A Pack's structure is as follows: EncryptedBlob1 || ... || EncryptedBlobN || EncryptedHeader || Header_Length -At the end of the Pack is a header, which describes the content. The header is -encrypted and authenticated. `Header_Length` is the length of the encrypted header -encoded as a four byte integer in little-endian encoding. Placing the header at -the end of a file allows writing the blobs in a continuous stream as soon as -they are read during the backup phase. This reduces code complexity and avoids -having to re-write a file once the pack is complete and the content and length -of the header is known. +At the end of the Pack file is a header, which describes the content. The +header is encrypted and authenticated. `Header_Length` is the length of the +encrypted header encoded as a four byte integer in little-endian encoding. +Placing the header at the end of a file allows writing the blobs in a +continuous stream as soon as they are read during the backup phase. This +reduces code complexity and avoids having to re-write a file once the pack is +complete and the content and length of the header is known. All the blobs (`EncryptedBlob1`, `EncryptedBlobN` etc.) are authenticated and encrypted independently. This enables repository reorganisation without having @@ -178,7 +192,7 @@ listed afterwards. There may be an arbitrary number of index files, containing information on non-disjoint sets of Packs. The number of packs described in a single file is -chosen so that the file size is kep below 8 MiB. +chosen so that the file size is kept below 8 MiB. Keys, Encryption and MAC ------------------------ @@ -230,9 +244,8 @@ tampered with, the computed MAC will not match the last 16 bytes of the data, and restic exits with an error. Otherwise, the data is decrypted with the encryption key derived from `scrypt`. This yields a JSON document which contains the master encryption and message authentication keys for this -repository (encoded in Base64) and the polynomial that is used for CDC. The -command `restic cat masterkey` can be used as follows to decrypt and -pretty-print the master key: +repository (encoded in Base64). The command `restic cat masterkey` can be used +as follows to decrypt and pretty-print the master key: $ restic -r /tmp/restic-repo cat masterkey { @@ -241,7 +254,6 @@ pretty-print the master key: "r": "E9eEDnSJZgqwTOkDtOp+Dw==" }, "encrypt": "UQCqa0lKZ94PygPxMRqkePTZnHRYh1k1pX2k2lM2v3Q=", - "chunker_polynomial": "2f0797d9c2363f" } All data in the repository is encrypted and authenticated with these master keys. @@ -284,9 +296,9 @@ hash. Before saving, each file is split into variable sized Blobs of data. The SHA-256 hashes of all Blobs are saved in an ordered list which then represents the content of the file. -In order to relate these plain text hashes to the actual encrypted storage -hashes (which vary due to random IVs), an index is used. If the index is not -available, the header of all data Blobs can be read. +In order to relate these plain text hashes to the actual location within a Pack +file , an index is used. If the index is not available, the header of all data +Blobs can be read. Trees and Data -------------- @@ -321,7 +333,7 @@ The command `restic cat tree` can be used to inspect the tree referenced above: A tree contains a list of entries (in the field `nodes`) which contain meta data like a name and timestamps. When the entry references a directory, the -field `subtree` contains the plain text ID of another tree object. +field `subtree` contains the plain text ID of another tree object. When the command `restic cat tree` is used, the storage hash is needed to print a tree. The tree referenced above can be dumped as follows: @@ -372,8 +384,9 @@ For creating a backup, restic scans the source directory for all files, sub-directories and other entries. The data from each file is split into variable length Blobs cut at offsets defined by a sliding window of 64 byte. The implementation uses Rabin Fingerprints for implementing this Content -Defined Chunking (CDC). An irreducible polynomial is selected at random when a -repository is initialized. +Defined Chunking (CDC). An irreducible polynomial is selected at random and +saved in the file `config` when a repository is initialized, so that watermark +attacks are much harder. Files smaller than 512 KiB are not split, Blobs are of 512 KiB to 8 MiB in size. The implementation aims for 1 MiB Blob size on average. From 1cedff0e9ea5d04817ec2b357a41d6be60eccdf2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 16:36:52 +0200 Subject: [PATCH 02/17] tests: Simplify key handling --- archiver_test.go | 20 ++++++-------------- cache_test.go | 2 -- server/key_test.go | 13 ------------- server/server.go | 13 +++++++++---- server/server_test.go | 14 -------------- test/backend.go | 15 +++++---------- tree_test.go | 2 -- walk_test.go | 2 -- 8 files changed, 20 insertions(+), 61 deletions(-) delete mode 100644 server/key_test.go diff --git a/archiver_test.go b/archiver_test.go index e0fbae084..dce0e0b7e 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -55,9 +55,8 @@ func BenchmarkChunkEncrypt(b *testing.B) { data := Random(23, 10<<20) // 10MiB rd := bytes.NewReader(data) - be := SetupBackend(b) - defer TeardownBackend(b, be) - key := SetupKey(b, be, "geheim") + s := SetupBackend(b) + defer TeardownBackend(b, s) buf := restic.GetChunkBuf("BenchmarkChunkEncrypt") buf2 := restic.GetChunkBuf("BenchmarkChunkEncrypt") @@ -66,7 +65,7 @@ func BenchmarkChunkEncrypt(b *testing.B) { b.SetBytes(int64(len(data))) for i := 0; i < b.N; i++ { - benchmarkChunkEncrypt(b, buf, buf2, rd, key) + benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key()) } restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf) @@ -94,9 +93,8 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *server.Key) } func BenchmarkChunkEncryptParallel(b *testing.B) { - be := SetupBackend(b) - defer TeardownBackend(b, be) - key := SetupKey(b, be, "geheim") + s := SetupBackend(b) + defer TeardownBackend(b, s) data := Random(23, 10<<20) // 10MiB @@ -108,7 +106,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { rd := bytes.NewReader(data) - benchmarkChunkEncryptP(pb, buf, rd, key) + benchmarkChunkEncryptP(pb, buf, rd, s.Key()) } }) @@ -118,8 +116,6 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { func archiveDirectory(b testing.TB) { server := SetupBackend(b) defer TeardownBackend(b, server) - key := SetupKey(b, server, "geheim") - server.SetKey(key) arch := restic.NewArchiver(server) @@ -154,8 +150,6 @@ func archiveWithDedup(t testing.TB) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) var cnt struct { before, after, after2 struct { @@ -236,8 +230,6 @@ func BenchmarkLoadTree(t *testing.B) { s := SetupBackend(t) defer TeardownBackend(t, s) - key := SetupKey(t, s, "geheim") - s.SetKey(key) // archive a few files arch := restic.NewArchiver(s) diff --git a/cache_test.go b/cache_test.go index 9948dbbfc..8540e1839 100644 --- a/cache_test.go +++ b/cache_test.go @@ -10,8 +10,6 @@ import ( func TestCache(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) _, err := restic.NewCache(server) OK(t, err) diff --git a/server/key_test.go b/server/key_test.go deleted file mode 100644 index 468e09c65..000000000 --- a/server/key_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package server_test - -import ( - "testing" - - . "github.com/restic/restic/test" -) - -func TestRepo(t *testing.T) { - s := SetupBackend(t) - defer TeardownBackend(t, s) - _ = SetupKey(t, s, TestPassword) -} diff --git a/server/server.go b/server/server.go index e2a84650c..accc1c40c 100644 --- a/server/server.go +++ b/server/server.go @@ -32,10 +32,6 @@ func NewServer(be backend.Backend) *Server { } } -func (s *Server) SetKey(k *Key) { - s.key = k -} - // ChunkerPolynomial returns the secret polynomial used for content defined chunking. func (s *Server) ChunkerPolynomial() chunker.Pol { return chunker.Pol(s.key.Master().ChunkerPolynomial) @@ -532,7 +528,16 @@ func (s *Server) SearchKey(password string) error { } s.key = key + return nil +} +func (s *Server) CreateKey(password string) error { + key, err := CreateKey(s, password) + if err != nil { + return err + } + + s.key = key return nil } diff --git a/server/server_test.go b/server/server_test.go index 51556ee94..55c22760c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -30,8 +30,6 @@ var serverTests = []testJSONStruct{ func TestSaveJSON(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) for _, obj := range serverTests { data, err := json.Marshal(obj) @@ -51,8 +49,6 @@ func TestSaveJSON(t *testing.T) { func BenchmarkSaveJSON(t *testing.B) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) obj := serverTests[0] @@ -78,8 +74,6 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} func TestSave(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) for _, size := range testSizes { data := make([]byte, size) @@ -112,8 +106,6 @@ func TestSave(t *testing.T) { func TestSaveFrom(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) for _, size := range testSizes { data := make([]byte, size) @@ -144,8 +136,6 @@ func TestSaveFrom(t *testing.T) { func BenchmarkSaveFrom(t *testing.B) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) size := 4 << 20 // 4MiB @@ -172,8 +162,6 @@ func TestLoadJSONPack(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) // archive a few files sn := SnapshotDir(t, server, *benchTestDir, nil) @@ -191,8 +179,6 @@ func TestLoadJSONEncrypted(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) // archive a snapshot sn := restic.Snapshot{} diff --git a/test/backend.go b/test/backend.go index 31be3f0ab..89bcb7b10 100644 --- a/test/backend.go +++ b/test/backend.go @@ -13,7 +13,7 @@ import ( "github.com/restic/restic/server" ) -var TestPassword = "foobar" +var TestPassword = flag.String("test.password", "", `use this password for repositories created during tests (default: "geheim")`) var TestCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)") var TestTempDir = flag.String("test.tempdir", "", "use this directory for temporary storage (default: system temp dir)") @@ -25,11 +25,13 @@ func SetupBackend(t testing.TB) *server.Server { b, err := local.Create(filepath.Join(tempdir, "repo")) OK(t, err) - // set cache dir + // set cache dir below temp dir err = os.Setenv("RESTIC_CACHE", filepath.Join(tempdir, "cache")) OK(t, err) - return server.NewServer(b) + s := server.NewServer(b) + OK(t, s.CreateKey(*TestPassword)) + return s } func TeardownBackend(t testing.TB, s *server.Server) { @@ -42,13 +44,6 @@ func TeardownBackend(t testing.TB, s *server.Server) { OK(t, s.Delete()) } -func SetupKey(t testing.TB, s *server.Server, password string) *server.Key { - k, err := server.CreateKey(s, password) - OK(t, err) - - return k -} - func SnapshotDir(t testing.TB, server *server.Server, path string, parent backend.ID) *restic.Snapshot { arch := restic.NewArchiver(server) sn, _, err := arch.Snapshot(nil, []string{path}, parent) diff --git a/tree_test.go b/tree_test.go index ef50e1b93..ecfefa620 100644 --- a/tree_test.go +++ b/tree_test.go @@ -95,8 +95,6 @@ func TestNodeComparison(t *testing.T) { func TestLoadTree(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) // save tree tree := restic.NewTree() diff --git a/walk_test.go b/walk_test.go index 88e2914b2..048af71c0 100644 --- a/walk_test.go +++ b/walk_test.go @@ -18,8 +18,6 @@ func TestWalkTree(t *testing.T) { server := SetupBackend(t) defer TeardownBackend(t, server) - key := SetupKey(t, server, "geheim") - server.SetKey(key) // archive a few files arch := restic.NewArchiver(server) From 2fb17838852cd58ebd41a8fdd6f4b651b25f1f23 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 16:43:27 +0200 Subject: [PATCH 03/17] backend/server: remove id and version from backend --- backend/interface.go | 13 +---- backend/local/local.go | 126 ++++------------------------------------- backend/paths.go | 6 +- backend/sftp/sftp.go | 123 ++++------------------------------------ cache.go | 4 +- server/server.go | 2 +- 6 files changed, 28 insertions(+), 246 deletions(-) diff --git a/backend/interface.go b/backend/interface.go index a20b0e751..63a3a95f8 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -11,10 +11,7 @@ const ( Lock = "lock" Snapshot = "snapshot" Index = "index" -) - -const ( - Version = 1 + Config = "config" ) // A Backend manages data stored somewhere. @@ -43,17 +40,9 @@ type Backend interface { // Close the backend Close() error - Identifier Lister } -type Identifier interface { - // ID returns a unique ID for a specific repository. This means restic can - // recognize repositories accessed via different methods (e.g. local file - // access and sftp). - ID() string -} - type Lister interface { // List returns a channel that yields all names of blobs of type t in // lexicographic order. A goroutine is started for this. If the channel diff --git a/backend/local/local.go b/backend/local/local.go index a50d2ef31..753e905fa 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -1,9 +1,6 @@ package local import ( - "crypto/rand" - "crypto/sha256" - "encoding/hex" "errors" "fmt" "io" @@ -11,7 +8,6 @@ import ( "os" "path/filepath" "sort" - "strings" "github.com/restic/restic/backend" ) @@ -19,9 +15,7 @@ import ( var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") type Local struct { - p string - ver uint - id string + p string } // Open opens the local backend at dir. @@ -36,68 +30,19 @@ func Open(dir string) (*Local, error) { filepath.Join(dir, backend.Paths.Temp), } - // test if all necessary dirs and files are there + // test if all necessary dirs are there for _, d := range items { if _, err := os.Stat(d); err != nil { return nil, fmt.Errorf("%s does not exist", d) } } - // read version file - f, err := os.Open(filepath.Join(dir, backend.Paths.Version)) - if err != nil { - return nil, fmt.Errorf("unable to read version file: %v\n", err) - } - - var version uint - n, err := fmt.Fscanf(f, "%d", &version) - if err != nil { - return nil, err - } - - if n != 1 { - return nil, errors.New("could not read version from file") - } - - err = f.Close() - if err != nil { - return nil, err - } - - // check version - if version != backend.Version { - return nil, fmt.Errorf("wrong version %d", version) - } - - // read ID - f, err = os.Open(filepath.Join(dir, backend.Paths.ID)) - if err != nil { - return nil, err - } - - buf, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - - id := strings.TrimSpace(string(buf)) - if err != nil { - return nil, err - } - - return &Local{p: dir, ver: version, id: id}, nil + return &Local{p: dir}, nil } // Create creates all the necessary files and directories for a new local -// backend at dir. +// backend at dir. Afterwards a new config blob should must created. func Create(dir string) (*Local, error) { - versionFile := filepath.Join(dir, backend.Paths.Version) - idFile := filepath.Join(dir, backend.Paths.ID) dirs := []string{ dir, filepath.Join(dir, backend.Paths.Data), @@ -108,15 +53,10 @@ func Create(dir string) (*Local, error) { filepath.Join(dir, backend.Paths.Temp), } - // test if files already exist - _, err := os.Lstat(versionFile) + // test if config file already exist + _, err := os.Lstat(backend.Paths.Config) if err == nil { - return nil, errors.New("version file already exists") - } - - _, err = os.Lstat(idFile) - if err == nil { - return nil, errors.New("id file already exists") + return nil, errors.New("config file already exists") } // test if directories already exist @@ -134,44 +74,6 @@ func Create(dir string) (*Local, error) { } } - // create version file - f, err := os.Create(versionFile) - if err != nil { - return nil, err - } - - _, err = fmt.Fprintf(f, "%d\n", backend.Version) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - - // create ID file - id := make([]byte, sha256.Size) - _, err = rand.Read(id) - if err != nil { - return nil, err - } - - f, err = os.Create(idFile) - if err != nil { - return nil, err - } - - _, err = fmt.Fprintln(f, hex.EncodeToString(id)) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - // open backend return Open(dir) } @@ -265,6 +167,10 @@ func (b *Local) Create() (backend.Blob, error) { // Construct path for given Type and name. func filename(base string, t backend.Type, name string) string { + if t == backend.Config { + return filepath.Join(base, "config") + } + return filepath.Join(dirname(base, t, name), name) } @@ -376,16 +282,6 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { return ch } -// Version returns the version of this local backend. -func (b *Local) Version() uint { - return b.ver -} - -// ID returns the ID of this local backend. -func (b *Local) ID() string { - return b.id -} - // Delete removes the repository and all files. func (b *Local) Delete() error { return os.RemoveAll(b.p) } diff --git a/backend/paths.go b/backend/paths.go index d697a6cc6..8e29e6950 100644 --- a/backend/paths.go +++ b/backend/paths.go @@ -10,8 +10,7 @@ var Paths = struct { Locks string Keys string Temp string - Version string - ID string + Config string }{ "data", "snapshots", @@ -19,8 +18,7 @@ var Paths = struct { "locks", "keys", "tmp", - "version", - "id", + "config", } // Default modes for file-based backends diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 5c850ce77..2207d897a 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -2,18 +2,15 @@ package sftp import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "errors" "fmt" "io" - "io/ioutil" "log" "os" "os/exec" "path/filepath" "sort" - "strings" "github.com/pkg/sftp" "github.com/restic/restic/backend" @@ -24,10 +21,8 @@ const ( ) type SFTP struct { - c *sftp.Client - p string - ver uint - id string + c *sftp.Client + p string cmd *exec.Cmd } @@ -81,8 +76,7 @@ func Open(dir string, program string, args ...string) (*SFTP, error) { filepath.Join(dir, backend.Paths.Index), filepath.Join(dir, backend.Paths.Locks), filepath.Join(dir, backend.Paths.Keys), - filepath.Join(dir, backend.Paths.Version), - filepath.Join(dir, backend.Paths.ID), + filepath.Join(dir, backend.Paths.Temp), } for _, d := range items { if _, err := sftp.c.Lstat(d); err != nil { @@ -90,64 +84,18 @@ func Open(dir string, program string, args ...string) (*SFTP, error) { } } - // read version file - f, err := sftp.c.Open(filepath.Join(dir, backend.Paths.Version)) - if err != nil { - return nil, fmt.Errorf("unable to read version file: %v\n", err) - } - - var version uint - n, err := fmt.Fscanf(f, "%d", &version) - if err != nil { - return nil, err - } - - if n != 1 { - return nil, errors.New("could not read version from file") - } - - err = f.Close() - if err != nil { - return nil, err - } - - // check version - if version != backend.Version { - return nil, fmt.Errorf("wrong version %d", version) - } - - // read ID - f, err = sftp.c.Open(filepath.Join(dir, backend.Paths.ID)) - if err != nil { - return nil, err - } - - buf, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - - sftp.id = strings.TrimSpace(string(buf)) sftp.p = dir - return sftp, nil } // Create creates all the necessary files and directories for a new sftp -// backend at dir. +// backend at dir. Afterwards a new config blob should must created. func Create(dir string, program string, args ...string) (*SFTP, error) { sftp, err := startClient(program, args...) if err != nil { return nil, err } - versionFile := filepath.Join(dir, backend.Paths.Version) - idFile := filepath.Join(dir, backend.Paths.ID) dirs := []string{ dir, filepath.Join(dir, backend.Paths.Data), @@ -158,15 +106,10 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { filepath.Join(dir, backend.Paths.Temp), } - // test if files already exist - _, err = sftp.c.Lstat(versionFile) + // test if config file already exist + _, err = sftp.c.Lstat(backend.Paths.Config) if err == nil { - return nil, errors.New("version file already exists") - } - - _, err = sftp.c.Lstat(idFile) - if err == nil { - return nil, errors.New("id file already exists") + return nil, errors.New("config file already exists") } // test if directories already exist @@ -184,44 +127,6 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { } } - // create version file - f, err := sftp.c.Create(versionFile) - if err != nil { - return nil, err - } - - _, err = fmt.Fprintf(f, "%d\n", backend.Version) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - - // create ID file - id := make([]byte, sha256.Size) - _, err = rand.Read(id) - if err != nil { - return nil, err - } - - f, err = sftp.c.Create(idFile) - if err != nil { - return nil, err - } - - _, err = fmt.Fprintln(f, hex.EncodeToString(id)) - if err != nil { - return nil, err - } - - err = f.Close() - if err != nil { - return nil, err - } - err = sftp.c.Close() if err != nil { return nil, err @@ -389,6 +294,10 @@ func (r *SFTP) Create() (backend.Blob, error) { // Construct path for given backend.Type and name. func (r *SFTP) filename(t backend.Type, name string) string { + if t == backend.Config { + return filepath.Join(r.p, "config") + } + return filepath.Join(r.dirname(t, name), name) } @@ -542,16 +451,6 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { } -// Version returns the version of this local backend. -func (r *SFTP) Version() uint { - return r.ver -} - -// ID returns the ID of this local backend. -func (r *SFTP) ID() string { - return r.id -} - // Close closes the sftp connection and terminates the underlying command. func (s *SFTP) Close() error { if s == nil { diff --git a/cache.go b/cache.go index 24f7a17fd..56fe3e321 100644 --- a/cache.go +++ b/cache.go @@ -18,13 +18,13 @@ type Cache struct { base string } -func NewCache(be backend.Identifier) (*Cache, error) { +func NewCache(s *server.Server) (*Cache, error) { cacheDir, err := getCacheDir() if err != nil { return nil, err } - basedir := filepath.Join(cacheDir, be.ID()) + basedir := filepath.Join(cacheDir, s.ID()) debug.Log("Cache.New", "opened cache at %v", basedir) return &Cache{base: basedir}, nil diff --git a/server/server.go b/server/server.go index accc1c40c..c616fbb0d 100644 --- a/server/server.go +++ b/server/server.go @@ -601,7 +601,7 @@ func (s *Server) Delete() error { } func (s *Server) ID() string { - return s.be.ID() + return "empty" } func (s *Server) Location() string { From 9b54fd7bdb7ee6b30224bd9bc20d63355674d279 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:14:12 +0200 Subject: [PATCH 04/17] server: rename LoadJSONEncrypted -> LoadJSONUnpacked --- cmd/restic/cmd_cat.go | 2 +- server/server.go | 2 +- server/server_test.go | 2 +- snapshot.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index c49b7d0e2..fb75a8ba3 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -78,7 +78,7 @@ func (cmd CmdCat) Execute(args []string) error { case "snapshot": sn := &restic.Snapshot{} - err = s.LoadJSONEncrypted(backend.Snapshot, id, sn) + err = s.LoadJSONUnpacked(backend.Snapshot, id, sn) if err != nil { return err } diff --git a/server/server.go b/server/server.go index c616fbb0d..c6e7fef63 100644 --- a/server/server.go +++ b/server/server.go @@ -143,7 +143,7 @@ func (s *Server) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { // LoadJSONEncrypted decrypts the data and afterwards calls json.Unmarshal on // the item. -func (s *Server) LoadJSONEncrypted(t backend.Type, id backend.ID, item interface{}) error { +func (s *Server) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error { // load blob from backend rd, err := s.be.Get(t, id.String()) if err != nil { diff --git a/server/server_test.go b/server/server_test.go index 55c22760c..83a41bb08 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -191,7 +191,7 @@ func TestLoadJSONEncrypted(t *testing.T) { var sn2 restic.Snapshot // restore - err = server.LoadJSONEncrypted(backend.Snapshot, id, &sn2) + err = server.LoadJSONUnpacked(backend.Snapshot, id, &sn2) OK(t, err) Equals(t, sn.Hostname, sn2.Hostname) diff --git a/snapshot.go b/snapshot.go index 6263d1a55..a928e27ad 100644 --- a/snapshot.go +++ b/snapshot.go @@ -52,7 +52,7 @@ func NewSnapshot(paths []string) (*Snapshot, error) { func LoadSnapshot(s *server.Server, id backend.ID) (*Snapshot, error) { sn := &Snapshot{id: id} - err := s.LoadJSONEncrypted(backend.Snapshot, id, sn) + err := s.LoadJSONUnpacked(backend.Snapshot, id, sn) if err != nil { return nil, err } From bebb08ee7edc46c3ef04f231dc7e2e8cca34ae6b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:17:10 +0200 Subject: [PATCH 05/17] server/key: Rename CreateKey -> CreateMasterKey --- cmd/restic/main.go | 2 +- server/key.go | 6 +++--- server/server.go | 6 ++++-- test/backend.go | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 806fcd0c1..5bbb8b586 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -75,7 +75,7 @@ func (cmd CmdInit) Execute(args []string) error { s := server.NewServer(be) - _, err = server.CreateKey(s, pw) + _, err = server.CreateMasterKey(s, pw) if err != nil { fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err) os.Exit(1) diff --git a/server/key.go b/server/key.go index 68f00c481..677870c87 100644 --- a/server/key.go +++ b/server/key.go @@ -50,9 +50,9 @@ type Key struct { name string } -// CreateKey initializes a master key in the given backend and encrypts it with -// the password. -func CreateKey(s *Server, password string) (*Key, error) { +// CreateMasterKey creates a new master key in the given backend and encrypts +// it with the password. +func CreateMasterKey(s *Server, password string) (*Key, error) { return AddKey(s, password, nil) } diff --git a/server/server.go b/server/server.go index c6e7fef63..198792adf 100644 --- a/server/server.go +++ b/server/server.go @@ -531,8 +531,10 @@ func (s *Server) SearchKey(password string) error { return nil } -func (s *Server) CreateKey(password string) error { - key, err := CreateKey(s, password) +// CreateMasterKey creates a new key with the supplied password, afterwards the +// repository config is created. +func (s *Server) CreateMasterKey(password string) error { + key, err := CreateMasterKey(s, password) if err != nil { return err } diff --git a/test/backend.go b/test/backend.go index 89bcb7b10..c7db75135 100644 --- a/test/backend.go +++ b/test/backend.go @@ -30,7 +30,7 @@ func SetupBackend(t testing.TB) *server.Server { OK(t, err) s := server.NewServer(b) - OK(t, s.CreateKey(*TestPassword)) + OK(t, s.CreateMasterKey(*TestPassword)) return s } From d4bf5bb279371df953927e3ea16e837cd679c204 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:37:12 +0200 Subject: [PATCH 06/17] server: Add config --- archiver.go | 2 +- cache.go | 2 +- cmd/restic/main.go | 5 ++-- server/key.go | 4 +-- server/server.go | 66 ++++++++++++++++++++++++++++++++++------------ 5 files changed, 55 insertions(+), 24 deletions(-) diff --git a/archiver.go b/archiver.go index ca8542b65..f77efb4ad 100644 --- a/archiver.go +++ b/archiver.go @@ -184,7 +184,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error { } chnker := GetChunker("archiver.SaveFile") - chnker.Reset(file, arch.s.ChunkerPolynomial()) + chnker.Reset(file, arch.s.Config.ChunkerPolynomial) resultChannels := [](<-chan saveResult){} defer FreeChunker("archiver.SaveFile", chnker) diff --git a/cache.go b/cache.go index 56fe3e321..a3c7b8953 100644 --- a/cache.go +++ b/cache.go @@ -24,7 +24,7 @@ func NewCache(s *server.Server) (*Cache, error) { return nil, err } - basedir := filepath.Join(cacheDir, s.ID()) + basedir := filepath.Join(cacheDir, s.Config.ID) debug.Log("Cache.New", "opened cache at %v", basedir) return &Cache{base: basedir}, nil diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 5bbb8b586..a6de28edf 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -74,14 +74,13 @@ func (cmd CmdInit) Execute(args []string) error { } s := server.NewServer(be) - - _, err = server.CreateMasterKey(s, pw) + err = s.CreateMasterKey(pw) if err != nil { fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err) os.Exit(1) } - fmt.Printf("created restic backend %v at %s\n", s.ID()[:10], opts.Repo) + fmt.Printf("created restic backend %v at %s\n", s.Config.ID[:10], opts.Repo) fmt.Println("Please note that knowledge of your password is required to access the repository.") fmt.Println("Losing your password means that your data is irrecoverably lost.") diff --git a/server/key.go b/server/key.go index 677870c87..1e4294984 100644 --- a/server/key.go +++ b/server/key.go @@ -50,9 +50,9 @@ type Key struct { name string } -// CreateMasterKey creates a new master key in the given backend and encrypts +// createMasterKey creates a new master key in the given backend and encrypts // it with the password. -func CreateMasterKey(s *Server, password string) (*Key, error) { +func createMasterKey(s *Server, password string) (*Key, error) { return AddKey(s, password, nil) } diff --git a/server/server.go b/server/server.go index 198792adf..be79727c4 100644 --- a/server/server.go +++ b/server/server.go @@ -2,7 +2,9 @@ package server import ( "bytes" + "crypto/rand" "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" @@ -16,10 +18,19 @@ import ( "github.com/restic/restic/pack" ) +// Config contains the configuration for a repository. +type Config struct { + Version uint `json:"version"` + ID string `json:"id"` + ChunkerPolynomial chunker.Pol `json:"chunker_polynomial"` +} + +// Server is used to access a repository in a backend. type Server struct { - be backend.Backend - key *Key - idx *Index + be backend.Backend + Config Config + key *Key + idx *Index pm sync.Mutex packs []*pack.Packer @@ -32,11 +43,6 @@ func NewServer(be backend.Backend) *Server { } } -// ChunkerPolynomial returns the secret polynomial used for content defined chunking. -func (s *Server) ChunkerPolynomial() chunker.Pol { - return chunker.Pol(s.key.Master().ChunkerPolynomial) -} - // Find loads the list of all blobs of type t and searches for names which start // with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If // more than one is found, nil and ErrMultipleIDMatches is returned. @@ -365,12 +371,12 @@ func (s *Server) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, error) // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the // backend as type t, without a pack. It returns the storage hash. func (s *Server) SaveJSONUnpacked(t backend.Type, item interface{}) (backend.ID, error) { - // create blob + // create file blob, err := s.be.Create() if err != nil { return nil, err } - debug.Log("Server.SaveJSONUnpacked", "create new pack %p", blob) + debug.Log("Server.SaveJSONUnpacked", "create new file %p", blob) // hash hw := backend.NewHashingWriter(blob, sha256.New()) @@ -521,6 +527,36 @@ func (s *Server) loadIndex(id string) error { return nil } +const repositoryIDSize = sha256.Size +const RepositoryVersion = 1 + +func (s *Server) createConfig() (err error) { + s.Config.ChunkerPolynomial, err = chunker.RandomPolynomial() + if err != nil { + return err + } + + newID := make([]byte, repositoryIDSize) + _, err = io.ReadFull(rand.Reader, newID) + if err != nil { + return err + } + + s.Config.ID = hex.EncodeToString(newID) + s.Config.Version = RepositoryVersion + + debug.Log("Server.createConfig", "New config: %#v", s.Config) + + _, err = s.SaveJSONUnpacked(backend.Config, s.Config) + return err +} + +func (s *Server) loadConfig(cfg *Config) error { + return s.LoadJSONUnpacked(backend.Config, nil, cfg) +} + +// SearchKey tries to find a key for which the supplied password works, +// afterwards the repository config is read and parsed. func (s *Server) SearchKey(password string) error { key, err := SearchKey(s, password) if err != nil { @@ -528,19 +564,19 @@ func (s *Server) SearchKey(password string) error { } s.key = key - return nil + return s.loadConfig(&s.Config) } // CreateMasterKey creates a new key with the supplied password, afterwards the // repository config is created. func (s *Server) CreateMasterKey(password string) error { - key, err := CreateMasterKey(s, password) + key, err := createMasterKey(s, password) if err != nil { return err } s.key = key - return nil + return s.createConfig() } func (s *Server) Decrypt(ciphertext []byte) ([]byte, error) { @@ -602,10 +638,6 @@ func (s *Server) Delete() error { return errors.New("Delete() called for backend that does not implement this method") } -func (s *Server) ID() string { - return "empty" -} - func (s *Server) Location() string { return s.be.Location() } From 765e3dc66f0cef36b8d48b0672609bb1995df052 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:37:21 +0200 Subject: [PATCH 07/17] restic: Add 'cat config' command --- cmd/restic/cmd_cat.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index fb75a8ba3..b4914645d 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -27,11 +27,11 @@ func init() { } func (cmd CmdCat) Usage() string { - return "[pack|blob|tree|snapshot|key|masterkey|lock] ID" + return "[pack|blob|tree|snapshot|key|masterkey|config|lock] ID" } func (cmd CmdCat) Execute(args []string) error { - if len(args) < 1 || (args[0] != "masterkey" && len(args) != 2) { + if len(args) < 1 || (args[0] != "masterkey" && args[0] != "config" && len(args) != 2) { return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage()) } @@ -43,7 +43,7 @@ func (cmd CmdCat) Execute(args []string) error { tpe := args[0] var id backend.ID - if tpe != "masterkey" { + if tpe != "masterkey" && tpe != "config" { id, err = backend.ParseID(args[1]) if err != nil { id = nil @@ -67,6 +67,14 @@ func (cmd CmdCat) Execute(args []string) error { // handle all types that don't need an index switch tpe { + case "config": + buf, err := json.MarshalIndent(s.Config, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf)) + return nil case "index": buf, err := s.Load(backend.Index, id) if err != nil { From ddc44ddfb17314b31880d05ac45f15d180b1b13e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:42:01 +0200 Subject: [PATCH 08/17] server: Check irreducible polynomial in config --- server/server.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/server.go b/server/server.go index be79727c4..95c6fcf2d 100644 --- a/server/server.go +++ b/server/server.go @@ -552,7 +552,16 @@ func (s *Server) createConfig() (err error) { } func (s *Server) loadConfig(cfg *Config) error { - return s.LoadJSONUnpacked(backend.Config, nil, cfg) + err := s.LoadJSONUnpacked(backend.Config, nil, cfg) + if err != nil { + return err + } + + if !cfg.ChunkerPolynomial.Irreducible() { + return errors.New("invalid chunker polynomial") + } + + return nil } // SearchKey tries to find a key for which the supplied password works, From 991a325cc583fb6fe97d281581611039d2fe2727 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:46:18 +0200 Subject: [PATCH 09/17] server: Test for existing config --- server/server.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/server.go b/server/server.go index 95c6fcf2d..22da63083 100644 --- a/server/server.go +++ b/server/server.go @@ -579,6 +579,14 @@ func (s *Server) SearchKey(password string) error { // CreateMasterKey creates a new key with the supplied password, afterwards the // repository config is created. func (s *Server) CreateMasterKey(password string) error { + has, err := s.Test(backend.Config, "") + if err != nil { + return err + } + if has { + return errors.New("repository master key and config already initialized") + } + key, err := createMasterKey(s, password) if err != nil { return err From 08fac28e738cef66933374c6be2246b77c009be6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 17:51:04 +0200 Subject: [PATCH 10/17] crypto: Remove polynomial from key --- crypto/crypto.go | 13 +++---------- server/key.go | 17 ----------------- 2 files changed, 3 insertions(+), 27 deletions(-) diff --git a/crypto/crypto.go b/crypto/crypto.go index 418d80f8a..3e7aedf07 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" - "github.com/restic/restic/chunker" "golang.org/x/crypto/poly1305" "golang.org/x/crypto/scrypt" ) @@ -35,12 +34,10 @@ var ( // Key holds encryption and message authentication keys for a repository. It is stored // encrypted and authenticated as a JSON data structure in the Data field of the Key -// structure. For the master key, the secret random polynomial used for content -// defined chunking is included. +// structure. type Key struct { - MAC MACKey `json:"mac"` - Encrypt EncryptionKey `json:"encrypt"` - ChunkerPolynomial chunker.Pol `json:"chunker_polynomial,omitempty"` + MAC MACKey `json:"mac"` + Encrypt EncryptionKey `json:"encrypt"` } type EncryptionKey [32]byte @@ -340,9 +337,5 @@ func KDF(N, R, P int, salt []byte, password string) (*Key, error) { // Valid tests if the key is valid. func (k *Key) Valid() bool { - if k.ChunkerPolynomial != 0 && !k.ChunkerPolynomial.Irreducible() { - return false - } - return k.Encrypt.Valid() && k.MAC.Valid() } diff --git a/server/key.go b/server/key.go index 1e4294984..097ff6051 100644 --- a/server/key.go +++ b/server/key.go @@ -12,9 +12,7 @@ import ( "time" "github.com/restic/restic/backend" - "github.com/restic/restic/chunker" "github.com/restic/restic/crypto" - "github.com/restic/restic/debug" ) var ( @@ -92,13 +90,6 @@ func OpenKey(s *Server, name string, password string) (*Key, error) { return nil, errors.New("Invalid key for repository") } - // test if the chunker polynomial is present in the master key - if k.master.ChunkerPolynomial == 0 { - return nil, errors.New("Polynomial for content defined chunking is zero") - } - - debug.Log("OpenKey", "Master keys loaded, polynomial %v", k.master.ChunkerPolynomial) - return k, nil } @@ -177,14 +168,6 @@ func AddKey(s *Server, password string, template *Key) (*Key, error) { if template == nil { // generate new random master keys newkey.master = crypto.NewRandomKey() - // generate random polynomial for cdc - p, err := chunker.RandomPolynomial() - if err != nil { - debug.Log("AddKey", "error generating new polynomial for cdc: %v", err) - return nil, err - } - debug.Log("AddKey", "generated new polynomial for cdc: %v", p) - newkey.master.ChunkerPolynomial = p } else { // copy master keys from old key newkey.master = template.master From 1213d87b1a032c65d5f59eff7e2012fb9cbb0f13 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 18:04:13 +0200 Subject: [PATCH 11/17] server: Only save crypto.Key At the moment, the server doesn't need the full server.Key (master and user key), just the master key. --- archiver_test.go | 10 +++++----- cmd/restic/cmd_cat.go | 2 +- cmd/restic/cmd_key.go | 6 +++--- server/key.go | 4 ++-- server/server.go | 38 +++++++++++++++++++++++--------------- 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/archiver_test.go b/archiver_test.go index dce0e0b7e..deead2ba1 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic" "github.com/restic/restic/backend" "github.com/restic/restic/chunker" + "github.com/restic/restic/crypto" "github.com/restic/restic/pack" - "github.com/restic/restic/server" . "github.com/restic/restic/test" ) @@ -24,7 +24,7 @@ type Rdr interface { io.ReaderAt } -func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *server.Key) { +func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) { ch := restic.GetChunker("BenchmarkChunkEncrypt") rd.Seek(0, 0) ch.Reset(rd, testPol) @@ -44,7 +44,7 @@ func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *server.K OK(b, err) Assert(b, uint(n) == chunk.Length, "invalid length: got %d, expected %d", n, chunk.Length) - _, err = key.Encrypt(buf2, buf) + _, err = crypto.Encrypt(key, buf2, buf) OK(b, err) } @@ -72,7 +72,7 @@ func BenchmarkChunkEncrypt(b *testing.B) { restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf2) } -func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *server.Key) { +func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) { ch := restic.GetChunker("BenchmarkChunkEncryptP") rd.Seek(0, 0) ch.Reset(rd, testPol) @@ -86,7 +86,7 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *server.Key) // reduce length of chunkBuf buf = buf[:chunk.Length] io.ReadFull(chunk.Reader(rd), buf) - key.Encrypt(buf, buf) + crypto.Encrypt(key, buf, buf) } restic.FreeChunker("BenchmarkChunkEncryptP", ch) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index b4914645d..8fa54ed27 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -121,7 +121,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "masterkey": - buf, err := json.MarshalIndent(s.Key().Master(), "", " ") + buf, err := json.MarshalIndent(s.Key(), "", " ") if err != nil { return err } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 9e87ff247..de58d50cb 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -42,7 +42,7 @@ func listKeys(s *server.Server) error { } var current string - if name == s.Key().Name() { + if name == s.KeyName() { current = "*" } else { current = " " @@ -75,7 +75,7 @@ func addKey(s *server.Server) error { } func deleteKey(s *server.Server, name string) error { - if name == s.Key().Name() { + if name == s.KeyName() { return errors.New("refusing to remove key currently used to access repository") } @@ -103,7 +103,7 @@ func changePassword(s *server.Server) error { } // remove old key - err = s.Remove(backend.Key, s.Key().Name()) + err = s.Remove(backend.Key, s.KeyName()) if err != nil { return err } diff --git a/server/key.go b/server/key.go index 097ff6051..121235dba 100644 --- a/server/key.go +++ b/server/key.go @@ -132,7 +132,7 @@ func LoadKey(s *Server, name string) (*Key, error) { } // AddKey adds a new key to an already existing repository. -func AddKey(s *Server, password string, template *Key) (*Key, error) { +func AddKey(s *Server, password string, template *crypto.Key) (*Key, error) { // fill meta data about key newkey := &Key{ Created: time.Now(), @@ -170,7 +170,7 @@ func AddKey(s *Server, password string, template *Key) (*Key, error) { newkey.master = crypto.NewRandomKey() } else { // copy master keys from old key - newkey.master = template.master + newkey.master = template } // encrypt master keys (as json) with user key diff --git a/server/server.go b/server/server.go index 22da63083..696dedee7 100644 --- a/server/server.go +++ b/server/server.go @@ -14,6 +14,7 @@ import ( "github.com/restic/restic/backend" "github.com/restic/restic/chunker" + "github.com/restic/restic/crypto" "github.com/restic/restic/debug" "github.com/restic/restic/pack" ) @@ -27,10 +28,11 @@ type Config struct { // Server is used to access a repository in a backend. type Server struct { - be backend.Backend - Config Config - key *Key - idx *Index + be backend.Backend + Config Config + key *crypto.Key + keyName string + idx *Index pm sync.Mutex packs []*pack.Packer @@ -158,7 +160,7 @@ func (s *Server) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{ defer rd.Close() // decrypt - decryptRd, err := s.key.DecryptFrom(rd) + decryptRd, err := crypto.DecryptFrom(s.key, rd) defer decryptRd.Close() if err != nil { return err @@ -191,7 +193,7 @@ func (s *Server) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) defer rd.Close() // decrypt - decryptRd, err := s.key.DecryptFrom(rd) + decryptRd, err := crypto.DecryptFrom(s.key, rd) defer decryptRd.Close() if err != nil { return err @@ -236,7 +238,7 @@ func (s *Server) findPacker(size uint) (*pack.Packer, error) { return nil, err } debug.Log("Server.findPacker", "create new pack %p", blob) - return pack.NewPacker(s.key.Master(), blob), nil + return pack.NewPacker(s.key, blob), nil } // insertPacker appends p to s.packs. @@ -382,7 +384,7 @@ func (s *Server) SaveJSONUnpacked(t backend.Type, item interface{}) (backend.ID, hw := backend.NewHashingWriter(blob, sha256.New()) // encrypt blob - ewr := s.key.EncryptTo(hw) + ewr := crypto.EncryptTo(s.key, hw) enc := json.NewEncoder(ewr) err = enc.Encode(item) @@ -454,7 +456,7 @@ func (s *Server) SaveIndex() (backend.ID, error) { hw := backend.NewHashingWriter(blob, sha256.New()) // encrypt blob - ewr := s.key.EncryptTo(hw) + ewr := crypto.EncryptTo(s.key, hw) err = s.idx.Encode(ewr) if err != nil { @@ -507,7 +509,7 @@ func (s *Server) loadIndex(id string) error { } // decrypt - decryptRd, err := s.key.DecryptFrom(rd) + decryptRd, err := crypto.DecryptFrom(s.key, rd) defer decryptRd.Close() if err != nil { return err @@ -572,7 +574,8 @@ func (s *Server) SearchKey(password string) error { return err } - s.key = key + s.key = key.Master() + s.keyName = key.Name() return s.loadConfig(&s.Config) } @@ -592,7 +595,8 @@ func (s *Server) CreateMasterKey(password string) error { return err } - s.key = key + s.key = key.Master() + s.keyName = key.Name() return s.createConfig() } @@ -601,7 +605,7 @@ func (s *Server) Decrypt(ciphertext []byte) ([]byte, error) { return nil, errors.New("key for server not set") } - return s.key.Decrypt(nil, ciphertext) + return crypto.Decrypt(s.key, nil, ciphertext) } func (s *Server) Encrypt(ciphertext, plaintext []byte) ([]byte, error) { @@ -609,13 +613,17 @@ func (s *Server) Encrypt(ciphertext, plaintext []byte) ([]byte, error) { return nil, errors.New("key for server not set") } - return s.key.Encrypt(ciphertext, plaintext) + return crypto.Encrypt(s.key, ciphertext, plaintext) } -func (s *Server) Key() *Key { +func (s *Server) Key() *crypto.Key { return s.key } +func (s *Server) KeyName() string { + return s.keyName +} + // Count returns the number of blobs of a given type in the backend. func (s *Server) Count(t backend.Type) (n uint) { for _ = range s.be.List(t, nil) { From 7af918cfddbd70ebd81f9531267f67bd5cf711c8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 3 May 2015 18:07:21 +0200 Subject: [PATCH 12/17] key: Remove unused convenience methods --- server/key.go | 41 ----------------------------------------- server/server.go | 4 ++-- 2 files changed, 2 insertions(+), 43 deletions(-) diff --git a/server/key.go b/server/key.go index 121235dba..1b0965b37 100644 --- a/server/key.go +++ b/server/key.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "os" "os/user" "time" @@ -212,46 +211,6 @@ func AddKey(s *Server, password string, template *crypto.Key) (*Key, error) { return newkey, nil } -// Encrypt encrypts and authenticates data with the master key. Stored in -// ciphertext is IV || Ciphertext || MAC. Returns the ciphertext, which is -// extended if necessary. -func (k *Key) Encrypt(ciphertext, plaintext []byte) ([]byte, error) { - return crypto.Encrypt(k.master, ciphertext, plaintext) -} - -// EncryptTo encrypts and authenticates data with the master key. The returned -// io.Writer writes IV || Ciphertext || MAC. -func (k *Key) EncryptTo(wr io.Writer) io.WriteCloser { - return crypto.EncryptTo(k.master, wr) -} - -// Decrypt verifes and decrypts the ciphertext with the master key. Ciphertext -// must be in the form IV || Ciphertext || MAC. -func (k *Key) Decrypt(plaintext, ciphertext []byte) ([]byte, error) { - return crypto.Decrypt(k.master, plaintext, ciphertext) -} - -// DecryptFrom verifies and decrypts the ciphertext read from rd and makes it -// available on the returned Reader. Ciphertext must be in the form IV || -// Ciphertext || MAC. In order to correctly verify the ciphertext, rd is -// drained, locally buffered and made available on the returned Reader -// afterwards. If a MAC verification failure is observed, it is returned -// immediately. -func (k *Key) DecryptFrom(rd io.Reader) (io.ReadCloser, error) { - return crypto.DecryptFrom(k.master, rd) -} - -// Master returns the master keys for this repository. Only included for -// debug purposes. -func (k *Key) Master() *crypto.Key { - return k.master -} - -// User returns the user keys for this key. Only included for debug purposes. -func (k *Key) User() *crypto.Key { - return k.user -} - func (k *Key) String() string { if k == nil { return "" diff --git a/server/server.go b/server/server.go index 696dedee7..d8cc6dd98 100644 --- a/server/server.go +++ b/server/server.go @@ -574,7 +574,7 @@ func (s *Server) SearchKey(password string) error { return err } - s.key = key.Master() + s.key = key.master s.keyName = key.Name() return s.loadConfig(&s.Config) } @@ -595,7 +595,7 @@ func (s *Server) CreateMasterKey(password string) error { return err } - s.key = key.Master() + s.key = key.master s.keyName = key.Name() return s.createConfig() } From be943eaf8ba0611be0d57766a7ef923d38857c1c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 May 2015 20:39:07 +0200 Subject: [PATCH 13/17] doc: Clarify storage ID --- doc/Design.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/doc/Design.md b/doc/Design.md index 4463fadb3..483ce3621 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -21,17 +21,16 @@ been backed up at some point in time. The state here means the content and meta data like the name and modification time for the file or the directory and its contents. -*Storage ID*: A storage ID is the hash of the content of a file stored in the +*Storage ID*: A storage ID is the SHA-256 hash of the content stored in the repository. This ID is needed in order to load the file from the repository. -The storage hash is the SHA-256 hash of the content. Repository Format ================= All data is stored in a restic repository. A repository is able to store data of several different types, which can later be requested based on an ID. This -so-called "storage ID" is the hash (SHA-256) of the content of a file. All -files in a repository are only written once and never modified afterwards. This +so-called "storage ID" is the SHA-256 hash of the content of a file. All files +in a repository are only written once and never modified afterwards. This allows accessing and even writing to the repository with multiple clients in parallel. Only the delete operation removes data from the repository. @@ -40,7 +39,7 @@ directories and files. Such repositories can be accessed locally on the same system or via the integrated SFTP client. The directory layout is the same for both access methods. This repository type is described in the following. -Repositories consists of several directories and a file called `config`. For +Repositories consist of several directories and a file called `config`. For all other files stored in the repository, the name for the file is the lower case hexadecimal representation of the storage ID, which is the SHA-256 hash of the file's contents. This allows easily checking all files for accidental @@ -269,9 +268,8 @@ Snapshots A snapshots represents a directory with all files and sub-directories at a given point in time. For each backup that is made, a new snapshot is created. A snapshot is a JSON document that is stored in an encrypted file below the -directory `snapshots` in the repository. The filename is the SHA-256 hash of -the (encrypted) contents. This string is unique and used within restic to -uniquely identify a snapshot. +directory `snapshots` in the repository. The filename is the storage ID. This +string is unique and used within restic to uniquely identify a snapshot. The command `restic cat snapshot` can be used as follows to decrypt and pretty-print the contents of a snapshot file: @@ -296,7 +294,7 @@ hash. Before saving, each file is split into variable sized Blobs of data. The SHA-256 hashes of all Blobs are saved in an ordered list which then represents the content of the file. -In order to relate these plain text hashes to the actual location within a Pack +In order to relate these plaintext hashes to the actual location within a Pack file , an index is used. If the index is not available, the header of all data Blobs can be read. @@ -367,9 +365,9 @@ This tree contains a file entry. This time, the `subtree` field is not present and the `content` field contains a list with one plain text SHA-256 hash. The command `restic cat data` can be used to extract and decrypt data given a -storage hash, e.g. for the data mentioned above: +plaintext ID, e.g. for the data mentioned above: - $ restic -r /tmp/restic-repo cat blob 00634c46e5f7c055c341acd1201cf8289cabe769f991d6e350f8cd8ce2a52ac3 | sha256sum + $ restic -r /tmp/restic-repo cat blob 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d | sha256sum enter password for repository: 50f77b3b4291e8411a027b9f9b9e64658181cc676ce6ba9958b95f268cb1109d - From 5399358272ce55e4196ad1e44beb110235f6e54b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 May 2015 20:39:45 +0200 Subject: [PATCH 14/17] Fix spelling errors in comments --- backend/local/local.go | 4 ++-- backend/sftp/sftp.go | 4 ++-- server/server.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 753e905fa..c4ad266dc 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -41,7 +41,7 @@ func Open(dir string) (*Local, error) { } // Create creates all the necessary files and directories for a new local -// backend at dir. Afterwards a new config blob should must created. +// backend at dir. Afterwards a new config blob should be created. func Create(dir string) (*Local, error) { dirs := []string{ dir, @@ -53,7 +53,7 @@ func Create(dir string) (*Local, error) { filepath.Join(dir, backend.Paths.Temp), } - // test if config file already exist + // test if config file already exists _, err := os.Lstat(backend.Paths.Config) if err == nil { return nil, errors.New("config file already exists") diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 2207d897a..0397066fd 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -89,7 +89,7 @@ func Open(dir string, program string, args ...string) (*SFTP, error) { } // Create creates all the necessary files and directories for a new sftp -// backend at dir. Afterwards a new config blob should must created. +// backend at dir. Afterwards a new config blob should be created. func Create(dir string, program string, args ...string) (*SFTP, error) { sftp, err := startClient(program, args...) if err != nil { @@ -106,7 +106,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { filepath.Join(dir, backend.Paths.Temp), } - // test if config file already exist + // test if config file already exists _, err = sftp.c.Lstat(backend.Paths.Config) if err == nil { return nil, errors.New("config file already exists") diff --git a/server/server.go b/server/server.go index d8cc6dd98..74021f93e 100644 --- a/server/server.go +++ b/server/server.go @@ -149,7 +149,7 @@ func (s *Server) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { return plain, nil } -// LoadJSONEncrypted decrypts the data and afterwards calls json.Unmarshal on +// LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on // the item. func (s *Server) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) error { // load blob from backend @@ -566,8 +566,8 @@ func (s *Server) loadConfig(cfg *Config) error { return nil } -// SearchKey tries to find a key for which the supplied password works, -// afterwards the repository config is read and parsed. +// SearchKey finds a key with the supplied password, afterwards the config is +// read and parsed. func (s *Server) SearchKey(password string) error { key, err := SearchKey(s, password) if err != nil { From ae1a85c89655606d27835e6c907641e6d7f74a07 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 May 2015 20:40:02 +0200 Subject: [PATCH 15/17] server: Check repository version --- server/server.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/server.go b/server/server.go index 74021f93e..e896c7dec 100644 --- a/server/server.go +++ b/server/server.go @@ -559,6 +559,10 @@ func (s *Server) loadConfig(cfg *Config) error { return err } + if cfg.Version != RepositoryVersion { + return errors.New("unsupported repository version") + } + if !cfg.ChunkerPolynomial.Irreducible() { return errors.New("invalid chunker polynomial") } From 35af933f2456f49bd4c9d48e635663e3945bc370 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 May 2015 20:40:17 +0200 Subject: [PATCH 16/17] server: Rename CreateMasterKey() to Init() --- cmd/restic/main.go | 2 +- server/server.go | 10 +++++----- server/server_test.go | 2 +- test/backend.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cmd/restic/main.go b/cmd/restic/main.go index a6de28edf..d5d7cd169 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -74,7 +74,7 @@ func (cmd CmdInit) Execute(args []string) error { } s := server.NewServer(be) - err = s.CreateMasterKey(pw) + err = s.Init(pw) if err != nil { fmt.Fprintf(os.Stderr, "creating key in backend at %s failed: %v\n", opts.Repo, err) os.Exit(1) diff --git a/server/server.go b/server/server.go index e896c7dec..c6fc9732c 100644 --- a/server/server.go +++ b/server/server.go @@ -532,7 +532,7 @@ func (s *Server) loadIndex(id string) error { const repositoryIDSize = sha256.Size const RepositoryVersion = 1 -func (s *Server) createConfig() (err error) { +func createConfig(s *Server) (err error) { s.Config.ChunkerPolynomial, err = chunker.RandomPolynomial() if err != nil { return err @@ -583,9 +583,9 @@ func (s *Server) SearchKey(password string) error { return s.loadConfig(&s.Config) } -// CreateMasterKey creates a new key with the supplied password, afterwards the -// repository config is created. -func (s *Server) CreateMasterKey(password string) error { +// Init creates a new master key with the supplied password and initializes the +// repository config. +func (s *Server) Init(password string) error { has, err := s.Test(backend.Config, "") if err != nil { return err @@ -601,7 +601,7 @@ func (s *Server) CreateMasterKey(password string) error { s.key = key.master s.keyName = key.Name() - return s.createConfig() + return createConfig(s) } func (s *Server) Decrypt(ciphertext []byte) ([]byte, error) { diff --git a/server/server_test.go b/server/server_test.go index 83a41bb08..0f44def1c 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -172,7 +172,7 @@ func TestLoadJSONPack(t *testing.T) { OK(t, err) } -func TestLoadJSONEncrypted(t *testing.T) { +func TestLoadJSONUnpacked(t *testing.T) { if *benchTestDir == "" { t.Skip("benchdir not set, skipping TestServerStats") } diff --git a/test/backend.go b/test/backend.go index c7db75135..5a2524425 100644 --- a/test/backend.go +++ b/test/backend.go @@ -30,7 +30,7 @@ func SetupBackend(t testing.TB) *server.Server { OK(t, err) s := server.NewServer(b) - OK(t, s.CreateMasterKey(*TestPassword)) + OK(t, s.Init(*TestPassword)) return s } From f8804d42659022b441dde3bec07dbd921d17fcfe Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 4 May 2015 21:41:57 +0200 Subject: [PATCH 17/17] chunker: Add benchmark for reducibility test --- chunker/polynomials_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/chunker/polynomials_test.go b/chunker/polynomials_test.go index 1fce6fd7d..edea1124e 100644 --- a/chunker/polynomials_test.go +++ b/chunker/polynomials_test.go @@ -276,6 +276,22 @@ func TestPolIrreducible(t *testing.T) { } } +func BenchmarkPolIrreducible(b *testing.B) { + // find first irreducible polynomial + var pol chunker.Pol + for _, test := range polIrredTests { + if test.irred { + pol = test.f + break + } + } + + for i := 0; i < b.N; i++ { + Assert(b, pol.Irreducible(), + "Irreducibility test for Polynomial %v failed", pol) + } +} + var polGCDTests = []struct { f1 chunker.Pol f2 chunker.Pol