From 5dc8d3588d90fdcf8928b7808293053891ccc5d9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jan 2018 20:12:34 +0100 Subject: [PATCH 1/3] GS: Use generic http transport During the development of #1524 I discovered that the Google Cloud Storage backend did not yet use the HTTP transport, so things such as bandwidth limiting did not work. This commit does the necessary magic to make the GS library use our HTTP transport. --- cmd/restic/global.go | 4 ++-- internal/backend/gs/gs.go | 28 ++++++++++++++++++++-------- internal/backend/gs/gs_test.go | 12 +++++++++--- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index fc916cbef..6055132bd 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -567,7 +567,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend, case "s3": be, err = s3.Open(cfg.(s3.Config), rt) case "gs": - be, err = gs.Open(cfg.(gs.Config)) + be, err = gs.Open(cfg.(gs.Config), rt) case "azure": be, err = azure.Open(cfg.(azure.Config), rt) case "swift": @@ -628,7 +628,7 @@ func create(s string, opts options.Options) (restic.Backend, error) { case "s3": return s3.Create(cfg.(s3.Config), rt) case "gs": - return gs.Create(cfg.(gs.Config)) + return gs.Create(cfg.(gs.Config), rt) case "azure": return azure.Create(cfg.(azure.Config), rt) case "swift": diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index e88d49f45..d273d3e71 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "path" "strings" @@ -16,6 +17,7 @@ import ( "io/ioutil" + "golang.org/x/oauth2" "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" storage "google.golang.org/api/storage/v1" @@ -41,7 +43,7 @@ type Backend struct { // Ensure that *Backend implements restic.Backend. var _ restic.Backend = &Backend{} -func getStorageService(jsonKeyPath string) (*storage.Service, error) { +func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) { raw, err := ioutil.ReadFile(jsonKeyPath) if err != nil { @@ -53,8 +55,18 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) { return nil, err } - client := conf.Client(context.TODO()) + // create a new HTTP client + httpClient := &http.Client{ + Transport: rt, + } + // create a now context with the HTTP client stored at the oauth2.HTTPClient key + ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient) + + // then pass this context to Client(), which returns a new HTTP client + client := conf.Client(ctx) + + // that we can then pass to New() service, err := storage.New(client) if err != nil { return nil, err @@ -65,10 +77,10 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) { const defaultListMaxItems = 1000 -func open(cfg Config) (*Backend, error) { +func open(cfg Config, rt http.RoundTripper) (*Backend, error) { debug.Log("open, config %#v", cfg) - service, err := getStorageService(cfg.JSONKeyPath) + service, err := getStorageService(cfg.JSONKeyPath, rt) if err != nil { return nil, errors.Wrap(err, "getStorageService") } @@ -95,8 +107,8 @@ func open(cfg Config) (*Backend, error) { } // Open opens the gs backend at the specified bucket. -func Open(cfg Config) (restic.Backend, error) { - return open(cfg) +func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + return open(cfg, rt) } // Create opens the gs backend at the specified bucket and attempts to creates @@ -104,8 +116,8 @@ func Open(cfg Config) (restic.Backend, error) { // // The service account must have the "storage.buckets.create" permission to // create a bucket the does not yet exist. -func Create(cfg Config) (restic.Backend, error) { - be, err := open(cfg) +func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) { + be, err := open(cfg, rt) if err != nil { return nil, errors.Wrap(err, "open") } diff --git a/internal/backend/gs/gs_test.go b/internal/backend/gs/gs_test.go index 1c55697f3..a7bf97848 100644 --- a/internal/backend/gs/gs_test.go +++ b/internal/backend/gs/gs_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/gs" "github.com/restic/restic/internal/backend/test" "github.com/restic/restic/internal/errors" @@ -15,6 +16,11 @@ import ( ) func newGSTestSuite(t testing.TB) *test.Suite { + tr, err := backend.Transport(backend.TransportOptions{}) + if err != nil { + t.Fatalf("cannot create transport for tests: %v", err) + } + return &test.Suite{ // do not use excessive data MinimalData: true, @@ -37,7 +43,7 @@ func newGSTestSuite(t testing.TB) *test.Suite { Create: func(config interface{}) (restic.Backend, error) { cfg := config.(gs.Config) - be, err := gs.Create(cfg) + be, err := gs.Create(cfg, tr) if err != nil { return nil, err } @@ -57,14 +63,14 @@ func newGSTestSuite(t testing.TB) *test.Suite { // OpenFn is a function that opens a previously created temporary repository. Open: func(config interface{}) (restic.Backend, error) { cfg := config.(gs.Config) - return gs.Open(cfg) + return gs.Open(cfg, tr) }, // CleanupFn removes data created during the tests. Cleanup: func(config interface{}) error { cfg := config.(gs.Config) - be, err := gs.Open(cfg) + be, err := gs.Open(cfg, tr) if err != nil { return err } From 649afb1198794cf5291ab4cbb28162767a550378 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jan 2018 20:16:51 +0100 Subject: [PATCH 2/3] Add entry to changelog --- changelog/0.8.2/pull-1594 | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/0.8.2/pull-1594 diff --git a/changelog/0.8.2/pull-1594 b/changelog/0.8.2/pull-1594 new file mode 100644 index 000000000..b160baa70 --- /dev/null +++ b/changelog/0.8.2/pull-1594 @@ -0,0 +1,7 @@ +Bugfix: Google Cloud Storage: Use generic HTTP transport + +It was discovered that the Google Cloud Storage backend did not use the generic +HTTP transport, so things such as bandwidth limiting with `--limit-upload` did +not work. This is resolved now. + +https://github.com/restic/restic/pull/1594 From 05f826a4fc764622f071612802b111a69464691c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jan 2018 21:48:41 +0100 Subject: [PATCH 3/3] Update Gopkg.lock --- Gopkg.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index 8448b2e68..a2858f831 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -231,6 +231,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a" + inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab" solver-name = "gps-cdcl" solver-version = 1