Merge pull request #1914 from restic/update-blazer

Add support for B2 application keys

Gopkg.lock

@@ -94,8 +94,8 @@
 [[projects]]
   name = "github.com/kurin/blazer"
   packages = ["b2","base","internal/b2assets","internal/b2types","internal/blog","x/window"]
-  revision = "7f1134c7489e86be5c924137996d4e421815f48a"
+  revision = "caf65aa76491dc533bac68ad3243ce72fa4e0a0a"
-  version = "v0.5.0"
+  version = "v0.5.1"
 
 [[projects]]
   name = "github.com/marstr/guid"

doc/030_preparing_a_new_repo.rst

@@ -293,16 +293,21 @@ Backblaze B2
 ************
 
 Restic can backup data to any Backblaze B2 bucket. You need to first setup the
-following environment variables with the credentials you obtained when signed
+following environment variables with the credentials you can find in the
-into your B2 account:
+dashboard in on the "Buckets" page when signed into your B2 account:
 
 .. code-block:: console
 
     $ export B2_ACCOUNT_ID=<MY_ACCOUNT_ID>
     $ export B2_ACCOUNT_KEY=<MY_SECRET_ACCOUNT_KEY>
 
-You can then easily initialize a repository stored at Backblaze B2. If the
+You can either specify the so-called "Master Application Key" here (which can
-bucket does not exist yet, it will be created:
+access any bucket at any path) or a dedicated "Application Key" created just
+for restic (which may be restricted to a specific bucket and/or path).
+
+You can then initialize a repository stored at Backblaze B2. If the
+bucket does not exist yet and the credentials you passed to restic have the
+privilege to create buckets, it will be created automatically:
 
 .. code-block:: console
 

vendor/github.com/kurin/blazer/b2/backend.go

@@ -154,6 +154,7 @@ type beKeyInterface interface {
 	name() string
 	expires() time.Time
 	secret() string
+	id() string
 }
 
 type beKey struct {
@@ -711,6 +712,7 @@ func (b *beKey) caps() []string                { return b.k.caps() }
 func (b *beKey) name() string                  { return b.k.name() }
 func (b *beKey) expires() time.Time            { return b.k.expires() }
 func (b *beKey) secret() string                { return b.k.secret() }
+func (b *beKey) id() string                    { return b.k.id() }
 
 func jitter(d time.Duration) time.Duration {
 	f := float64(d)

vendor/github.com/kurin/blazer/b2/baseline.go

@@ -105,6 +105,7 @@ type b2KeyInterface interface {
 	name() string
 	expires() time.Time
 	secret() string
+	id() string
 }
 
 type b2Root struct {
@@ -508,3 +509,4 @@ func (b *b2Key) caps() []string                { return b.b.Capabilities }
 func (b *b2Key) name() string                  { return b.b.Name }
 func (b *b2Key) expires() time.Time            { return b.b.Expires }
 func (b *b2Key) secret() string                { return b.b.Secret }
+func (b *b2Key) id() string                    { return b.b.ID }

vendor/github.com/kurin/blazer/b2/integration_test.go

@@ -1014,6 +1014,68 @@ func TestVerifyReader(t *testing.T) {
 	}
 }
 
+func TestListBucketsWithKey(t *testing.T) {
+	ctx := context.Background()
+	bucket, done := startLiveTest(ctx, t)
+	defer done()
+
+	key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client, err := NewClient(ctx, key.ID(), key.Secret())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := client.Bucket(ctx, bucket.Name()); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestListBucketContentsWithKey(t *testing.T) {
+	ctx := context.Background()
+	bucket, done := startLiveTest(ctx, t)
+	defer done()
+
+	for _, path := range []string{"foo/bar", "foo/baz", "foo", "bar", "baz"} {
+		if _, _, err := writeFile(ctx, bucket, path, 1, 1e8); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets", "listFiles"), Prefix("foo/"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	client, err := NewClient(ctx, key.ID(), key.Secret())
+	if err != nil {
+		t.Fatal(err)
+	}
+	obucket, err := client.Bucket(ctx, bucket.Name())
+	if err != nil {
+		t.Fatal(err)
+	}
+	iter := obucket.List(ctx)
+	var got []string
+	for iter.Next() {
+		got = append(got, iter.Object().Name())
+	}
+	if iter.Err() != nil {
+		t.Fatal(iter.Err())
+	}
+	want := []string{"foo/bar", "foo/baz"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("error listing objects with restricted key: got %v, want %v", got, want)
+	}
+	iter2 := obucket.List(ctx, ListHidden())
+	for iter2.Next() {
+	}
+	if iter2.Err() != nil {
+		t.Error(iter2.Err())
+	}
+}
+
 func TestCreateDeleteKey(t *testing.T) {
 	ctx := context.Background()
 	bucket, done := startLiveTest(ctx, t)
@@ -1049,9 +1111,7 @@ func TestCreateDeleteKey(t *testing.T) {
 
 	for _, e := range table {
 		var opts []KeyOption
-		for _, cap := range e.cap {
+		opts = append(opts, Capabilities(e.cap...))
-			opts = append(opts, Capability(cap))
-		}
 		if e.d != 0 {
 			opts = append(opts, Lifetime(e.d))
 		}

vendor/github.com/kurin/blazer/b2/key.go

@@ -47,6 +47,10 @@ func (k *Key) Delete(ctx context.Context) error { return k.k.del(ctx) }
 // operations.
 func (k *Key) Secret() string { return k.k.secret() }
 
+// ID returns the application key ID.  This, plus the secret, is necessary to
+// authenticate to B2.
+func (k *Key) ID() string { return k.k.id() }
+
 type keyOptions struct {
 	caps     []string
 	prefix   string
@@ -69,10 +73,10 @@ func Deadline(t time.Time) KeyOption {
 	return Lifetime(d)
 }
 
-// Capability requests a key with the given capability.
+// Capabilities requests a key with the given capability.
-func Capability(cap string) KeyOption {
+func Capabilities(caps ...string) KeyOption {
 	return func(k *keyOptions) {
-		k.caps = append(k.caps, cap)
+		k.caps = append(k.caps, caps...)
 	}
 }
 

vendor/github.com/kurin/blazer/base/base.go

@@ -42,7 +42,7 @@ import (
 
 const (
 	APIBase          = "https://api.backblazeb2.com"
-	DefaultUserAgent = "blazer/0.5.0"
+	DefaultUserAgent = "blazer/0.5.1"
 )
 
 type b2err struct {
@@ -268,6 +268,8 @@ type B2 struct {
 	downloadURI string
 	minPartSize int
 	opts        *b2Options
+	bucket      string // restricted to this bucket if present
+	pfx         string // restricted to objects with this prefix if present
 }
 
 // Update replaces the B2 object with a new one, in-place.
@@ -428,6 +430,8 @@ func AuthorizeAccount(ctx context.Context, account, key string, opts ...AuthOpti
 		apiURI:      b2resp.URI,
 		downloadURI: b2resp.DownloadURI,
 		minPartSize: b2resp.PartSize,
+		bucket:      b2resp.Allowed.Bucket,
+		pfx:         b2resp.Allowed.Prefix,
 		opts:        b2opts,
 	}, nil
 }
@@ -614,6 +618,7 @@ func (b *Bucket) BaseURL() string {
 func (b *B2) ListBuckets(ctx context.Context) ([]*Bucket, error) {
 	b2req := &b2types.ListBucketsRequest{
 		AccountID: b.accountID,
+		Bucket:    b.bucket,
 	}
 	b2resp := &b2types.ListBucketsResponse{}
 	headers := map[string]string{
@@ -967,6 +972,9 @@ func (b *Bucket) ListUnfinishedLargeFiles(ctx context.Context, count int, contin
 
 // ListFileNames wraps b2_list_file_names.
 func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, prefix, delimiter string) ([]*File, string, error) {
+	if prefix == "" {
+		prefix = b.b2.pfx
+	}
 	b2req := &b2types.ListFileNamesRequest{
 		Count:        count,
 		Continuation: continuation,
@@ -1007,6 +1015,9 @@ func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, pre
 
 // ListFileVersions wraps b2_list_file_versions.
 func (b *Bucket) ListFileVersions(ctx context.Context, count int, startName, startID, prefix, delimiter string) ([]*File, string, string, error) {
+	if prefix == "" {
+		prefix = b.b2.pfx
+	}
 	b2req := &b2types.ListFileVersionsRequest{
 		BucketID:  b.ID,
 		Count:     count,

vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go

@@ -65,9 +65,7 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{})
 	if *c.pfx != "" {
 		opts = append(opts, b2.Prefix(*c.pfx))
 	}
-	for _, c := range caps {
+	opts = append(opts, b2.Capabilities(caps...))
-		opts = append(opts, b2.Capability(c))
-	}
 
 	client, err := b2.NewClient(ctx, id, key, b2.UserAgent("b2keys"))
 	if err != nil {
@@ -86,10 +84,12 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{})
 		cr = bucket
 	}
 
-	if _, err := cr.CreateKey(ctx, name, opts...); err != nil {
+	b2key, err := cr.CreateKey(ctx, name, opts...)
+	if err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		return subcommands.ExitFailure
 	}
+	fmt.Printf("key=%s, secret=%s\n", b2key.ID(), b2key.Secret())
 	return subcommands.ExitSuccess
 }
 

vendor/github.com/kurin/blazer/internal/b2types/b2types.go

@@ -29,14 +29,20 @@ type ErrorMessage struct {
 }
 
 type AuthorizeAccountResponse struct {
-	AccountID      string   `json:"accountId"`
+	AccountID      string    `json:"accountId"`
-	AuthToken      string   `json:"authorizationToken"`
+	AuthToken      string    `json:"authorizationToken"`
-	URI            string   `json:"apiUrl"`
+	URI            string    `json:"apiUrl"`
-	DownloadURI    string   `json:"downloadUrl"`
+	DownloadURI    string    `json:"downloadUrl"`
-	MinPartSize    int      `json:"minimumPartSize"`
+	MinPartSize    int       `json:"minimumPartSize"`
-	PartSize       int      `json:"recommendedPartSize"`
+	PartSize       int       `json:"recommendedPartSize"`
-	AbsMinPartSize int      `json:"absoluteMinimumPartSize"`
+	AbsMinPartSize int       `json:"absoluteMinimumPartSize"`
-	Capabilities   []string `json:"capabilities"`
+	Allowed        Allowance `json:"allowed"`
+}
+
+type Allowance struct {
+	Capabilities []string `json:"capabilities"`
+	Bucket       string   `json:"bucketId"`
+	Prefix       string   `json:"namePrefix"`
 }
 
 type LifecycleRule struct {
@@ -69,6 +75,7 @@ type DeleteBucketRequest struct {
 
 type ListBucketsRequest struct {
 	AccountID string `json:"accountId"`
+	Bucket    string `json:"bucketId,omitempty"`
 }
 
 type ListBucketsResponse struct {

vendor/github.com/kurin/blazer/internal/pyre/api.go

@@ -255,9 +255,9 @@ func (s *Server) ListFileVersions(ctx context.Context, req *pb.ListFileVersionsR
 	return nil, nil
 }
 
-//type objTuple struct {
+type objTuple struct {
-//	name, version string
+	name, version string
-//}
+}
 
 type ListManager interface {
 	// NextN returns the next n objects, sorted by lexicographical order by name,
@@ -276,6 +276,35 @@ type VersionedObject interface {
 	NextNVersions(begin string, n int) ([]string, error)
 }
 
+func getDirNames(lm ListManager, bucket, name, prefix, delim string, n int) ([]string, error) {
+	var sfx string
+	var out []string
+	for n > 0 {
+		vo, err := lm.NextN(bucket, name, prefix, sfx, 1)
+		if err != nil {
+			return nil, err
+		}
+		if len(vo) == 0 {
+			return out, nil
+		}
+		v := vo[0]
+		name = v.Name()
+		suffix := name[len(prefix):]
+		i := strings.Index(suffix, delim)
+		if i < 0 {
+			sfx = ""
+			out = append(out, name)
+			name += "\000"
+			n--
+			continue
+		}
+		sfx = v.Name()[:len(prefix)+i+1]
+		out = append(out, sfx)
+		n--
+	}
+	return out, nil
+}
+
 //func getNextObjects(lm ListManager, bucket, name, prefix, delimiter string, n int) ([]VersionedObject, error) {
 //	if delimiter == "" {
 //		return lm.NextN(bucket, name, prefix, "", n)

vendor/github.com/kurin/blazer/internal/pyre/api_test.go

@@ -0,0 +1,137 @@
+// Copyright 2018, Google
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pyre
+
+import (
+	"reflect"
+	"sort"
+	"strings"
+	"sync"
+	"testing"
+)
+
+type testVersionedObject struct {
+	name     string
+	versions []string
+}
+
+func (t testVersionedObject) Name() string { return t.name }
+
+func (t testVersionedObject) NextNVersions(b string, n int) ([]string, error) {
+	var out []string
+	var seen bool
+	if b == "" {
+		seen = true
+	}
+	for _, v := range t.versions {
+		if b == v {
+			seen = true
+		}
+		if !seen {
+			continue
+		}
+		if len(out) >= n {
+			return out, nil
+		}
+		out = append(out, v)
+	}
+	return out, nil
+}
+
+type testListManager struct {
+	objs map[string][]string
+	m    sync.Mutex
+}
+
+func (t *testListManager) NextN(b, fn, pfx, spfx string, n int) ([]VersionedObject, error) {
+	t.m.Lock()
+	defer t.m.Unlock()
+
+	var out []VersionedObject
+	var keys []string
+	for k := range t.objs {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		if k < fn {
+			continue
+		}
+		if !strings.HasPrefix(k, pfx) {
+			continue
+		}
+		if spfx != "" && strings.HasPrefix(k, spfx) {
+			continue
+		}
+		out = append(out, testVersionedObject{name: k, versions: t.objs[k]})
+		n--
+		if n <= 0 {
+			return out, nil
+		}
+	}
+	return out, nil
+}
+
+func TestGetDirNames(t *testing.T) {
+	table := []struct {
+		lm    ListManager
+		name  string
+		pfx   string
+		delim string
+		num   int
+		want  []string
+	}{
+		{
+			lm: &testListManager{
+				objs: map[string][]string{
+					"/usr/local/etc/foo/bar": {"a"},
+					"/usr/local/etc/foo/baz": {"a"},
+					"/usr/local/etc/foo":     {"a"},
+					"/usr/local/etc/fool":    {"a"},
+				},
+			},
+			num:   2,
+			pfx:   "/usr/local/etc/",
+			delim: "/",
+			want:  []string{"/usr/local/etc/foo", "/usr/local/etc/foo/"},
+		},
+		{
+			lm: &testListManager{
+				objs: map[string][]string{
+					"/usr/local/etc/foo/bar": {"a"},
+					"/usr/local/etc/foo/baz": {"a"},
+					"/usr/local/etc/foo":     {"a"},
+					"/usr/local/etc/fool":    {"a"},
+					"/usr/local/etc/bar":     {"a"},
+				},
+			},
+			num:   4,
+			pfx:   "/usr/local/etc/",
+			delim: "/",
+			want:  []string{"/usr/local/etc/bar", "/usr/local/etc/foo", "/usr/local/etc/foo/", "/usr/local/etc/fool"},
+		},
+	}
+
+	for _, e := range table {
+		got, err := getDirNames(e.lm, "", e.name, e.pfx, e.delim, e.num)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if !reflect.DeepEqual(got, e.want) {
+			t.Errorf("getDirNames(%v, %q, %q, %q, %d): got %v, want %v", e.lm, e.name, e.pfx, e.delim, e.num, got, e.want)
+		}
+	}
+}