From 0249c16b0458602c09da62f7955bb6f50a6a047c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 May 2017 19:41:55 +0200 Subject: [PATCH 1/6] Revert "minio: Apply fix" This reverts commit f43d34899d6f7c491a7a07b0f263f10e6018559d. --- vendor/src/github.com/minio/minio-go/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/src/github.com/minio/minio-go/api.go b/vendor/src/github.com/minio/minio-go/api.go index 19964b71c..a563a18d4 100644 --- a/vendor/src/github.com/minio/minio-go/api.go +++ b/vendor/src/github.com/minio/minio-go/api.go @@ -553,7 +553,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Bucket region if set in error response and the error // code dictates invalid region, we can retry the request // with the new region. - if errResponse.Region != "" && res.StatusCode == http.StatusBadRequest { + if errResponse.Code == "InvalidRegion" && errResponse.Region != "" { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } From c7209ef2319f0ec2e8d0fd49df38d9a6d0d089a0 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Sat, 13 May 2017 19:53:24 +0200 Subject: [PATCH 2/6] api: Failed call retry with region only when http.StatusBadRequest. Fixes https://github.com/minio/minio-go/issues/677 --- vendor/src/github.com/minio/minio-go/api-put-bucket.go | 2 +- .../github.com/minio/minio-go/api-put-object-common.go | 10 ++++++---- vendor/src/github.com/minio/minio-go/api.go | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/vendor/src/github.com/minio/minio-go/api-put-bucket.go b/vendor/src/github.com/minio/minio-go/api-put-bucket.go index 001da6de3..ab3f355f5 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-bucket.go +++ b/vendor/src/github.com/minio/minio-go/api-put-bucket.go @@ -89,7 +89,7 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { if resp.StatusCode != http.StatusOK { err := httpRespToErrorResponse(resp, bucketName, "") errResp := ToErrorResponse(err) - if errResp.Code == "InvalidRegion" && errResp.Region != "" { + if resp.StatusCode == http.StatusBadRequest && errResp.Region != "" { // Fetch bucket region found in headers // of S3 error response, attempt bucket // create again. diff --git a/vendor/src/github.com/minio/minio-go/api-put-object-common.go b/vendor/src/github.com/minio/minio-go/api-put-object-common.go index 68a459f4a..d69447317 100644 --- a/vendor/src/github.com/minio/minio-go/api-put-object-common.go +++ b/vendor/src/github.com/minio/minio-go/api-put-object-common.go @@ -157,11 +157,13 @@ func hashCopyN(hashAlgorithms map[string]hash.Hash, hashSums map[string][]byte, return 0, err } } - - for k, v := range hashAlgorithms { - hashSums[k] = v.Sum(nil) + if err == nil && size == partSize { + for k, v := range hashAlgorithms { + hashSums[k] = v.Sum(nil) + } + return size, nil } - return size, err + return 0, ErrUnexpectedEOF(size, partSize, "", "") } // getUploadID - fetch upload id if already present for an object name diff --git a/vendor/src/github.com/minio/minio-go/api.go b/vendor/src/github.com/minio/minio-go/api.go index a563a18d4..a7ff96a03 100644 --- a/vendor/src/github.com/minio/minio-go/api.go +++ b/vendor/src/github.com/minio/minio-go/api.go @@ -553,7 +553,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Bucket region if set in error response and the error // code dictates invalid region, we can retry the request // with the new region. - if errResponse.Code == "InvalidRegion" && errResponse.Region != "" { + if res.StatusCode == http.StatusBadRequest && errResponse.Region != "" { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } From 4ac0d3ad4012ac09da5e77049c3d159ca4871319 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 May 2017 19:56:11 +0200 Subject: [PATCH 3/6] backend tests load: Use reader with Size() method --- src/restic/backend/test/tests.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index 4519ee25c..65b7d8978 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -277,7 +277,8 @@ func BackendTestLoad(t testing.TB, s *Suite) { type errorCloser struct { io.Reader - t testing.TB + size int64 + t testing.TB } func (ec errorCloser) Close() error { @@ -285,6 +286,10 @@ func (ec errorCloser) Close() error { return errors.New("forbidden method close was called") } +func (ec errorCloser) Size() int64 { + return ec.size +} + // BackendTestSave tests saving data in the backend. func BackendTestSave(t testing.TB, s *Suite) { b := s.open(t) @@ -354,7 +359,7 @@ func BackendTestSave(t testing.TB, s *Suite) { // wrap the tempfile in an errorCloser, so we can detect if the backend // closes the reader - err = b.Save(h, errorCloser{t: t, Reader: tmpfile}) + err = b.Save(h, errorCloser{t: t, size: int64(length), Reader: tmpfile}) if err != nil { t.Fatal(err) } From 5c6ec787898360a76d9cc347866f9369fbafd3fc Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 May 2017 19:58:25 +0200 Subject: [PATCH 4/6] s3: Remove log line --- src/restic/backend/s3/s3_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/restic/backend/s3/s3_test.go b/src/restic/backend/s3/s3_test.go index f50bfd680..22710f92e 100644 --- a/src/restic/backend/s3/s3_test.go +++ b/src/restic/backend/s3/s3_test.go @@ -169,7 +169,6 @@ func TestBackendMinio(t *testing.T) { cfg.stopServer() } if cfg.removeTempdir != nil { - t.Logf("removeTempdir %v", config) cfg.removeTempdir() } return nil From ee68f9298bab6a2b8951d51dfb45d957e9ccd85a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 13 May 2017 20:41:04 +0200 Subject: [PATCH 5/6] tests: Ensure that backend tests cannot be skipped on Travis --- run_integration_tests.go | 9 +++++++ src/restic/backend/rest/rest_test.go | 8 ++++++- src/restic/backend/s3/s3_test.go | 12 ++++++++++ src/restic/backend/sftp/sftp_test.go | 8 ++++++- src/restic/test/vars.go | 35 ++++++++++++++++++++-------- 5 files changed, 60 insertions(+), 12 deletions(-) diff --git a/run_integration_tests.go b/run_integration_tests.go index 00fe598fc..f820d49a9 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -150,6 +150,15 @@ func (env *TravisEnvironment) RunTests() error { env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor") + // ensure that the following tests cannot be silently skipped on Travis + ensureTests := []string{ + "restic/backend/rest.TestBackendREST", + "restic/backend/sftp.TestBackendSFTP", + "restic/backend/s3.TestBackendMinio", + "restic/backend/s3.TestBackendS3", + } + env.env["RESTIC_TEST_DISALLOW_SKIP"] = strings.Join(ensureTests, ",") + if *runCrossCompile { // compile for all target architectures with tags for _, tags := range []string{"release", "debug"} { diff --git a/src/restic/backend/rest/rest_test.go b/src/restic/backend/rest/rest_test.go index d378a364f..e8ed57646 100644 --- a/src/restic/backend/rest/rest_test.go +++ b/src/restic/backend/rest/rest_test.go @@ -60,7 +60,13 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() { } } -func TestBackend(t *testing.T) { +func TestBackendREST(t *testing.T) { + defer func() { + if t.Skipped() { + SkipDisallowed(t, "restic/backend/rest.TestBackendREST") + } + }() + ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/src/restic/backend/s3/s3_test.go b/src/restic/backend/s3/s3_test.go index 22710f92e..b6859bcd4 100644 --- a/src/restic/backend/s3/s3_test.go +++ b/src/restic/backend/s3/s3_test.go @@ -97,6 +97,12 @@ func newCredentials(t testing.TB) (key, secret string) { } func TestBackendMinio(t *testing.T) { + defer func() { + if t.Skipped() { + SkipDisallowed(t, "restic/backend/s3.TestBackendMinio") + } + }() + // try to find a minio binary _, err := exec.LookPath("minio") if err != nil { @@ -179,6 +185,12 @@ func TestBackendMinio(t *testing.T) { } func TestBackendS3(t *testing.T) { + defer func() { + if t.Skipped() { + SkipDisallowed(t, "restic/backend/s3.TestBackendS3") + } + }() + vars := []string{ "RESTIC_TEST_S3_KEY", "RESTIC_TEST_S3_SECRET", diff --git a/src/restic/backend/sftp/sftp_test.go b/src/restic/backend/sftp/sftp_test.go index dd9728676..8fb46d2a1 100644 --- a/src/restic/backend/sftp/sftp_test.go +++ b/src/restic/backend/sftp/sftp_test.go @@ -29,7 +29,13 @@ func findSFTPServerBinary() string { var sftpServer = findSFTPServerBinary() -func TestBackend(t *testing.T) { +func TestBackendSFTP(t *testing.T) { + defer func() { + if t.Skipped() { + SkipDisallowed(t, "restic/backend/sftp.TestBackendSFTP") + } + }() + if sftpServer == "" { t.Skip("sftp server binary not found") } diff --git a/src/restic/test/vars.go b/src/restic/test/vars.go index 908e4bf6b..616f969f6 100644 --- a/src/restic/test/vars.go +++ b/src/restic/test/vars.go @@ -3,19 +3,22 @@ package test import ( "fmt" "os" + "strings" + "testing" ) var ( - TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") - TestCleanupTempDirs = getBoolVar("RESTIC_TEST_CLEANUP", true) - TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") - RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) - RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true) - TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec") - TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") - BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") - TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "") - TestRESTServer = getStringVar("RESTIC_TEST_REST_SERVER", "") + TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") + TestCleanupTempDirs = getBoolVar("RESTIC_TEST_CLEANUP", true) + TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") + RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) + RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true) + TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec") + TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") + BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") + TestS3Server = getStringVar("RESTIC_TEST_S3_SERVER", "") + TestRESTServer = getStringVar("RESTIC_TEST_REST_SERVER", "") + TestIntegrationDisallowSkip = getStringVar("RESTIC_TEST_DISALLOW_SKIP", "") ) func getStringVar(name, defaultValue string) string { @@ -40,3 +43,15 @@ func getBoolVar(name string, defaultValue bool) bool { return defaultValue } + +// SkipDisallowed fails the test if it needs to run. The environment +// variable RESTIC_TEST_DISALLOW_SKIP contains a comma-separated list of test +// names that must be run. If name is in this list, the test is marked as +// failed. +func SkipDisallowed(t testing.TB, name string) { + for _, s := range strings.Split(TestIntegrationDisallowSkip, ",") { + if s == name { + t.Fatalf("test %v is in list of tests that need to run ($RESTIC_TEST_DISALLOW_SKIP)", name) + } + } +} From 82e15dc6dc85e4d02eb7eabbb3c53d87383e9e04 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 14 May 2017 00:29:10 +0200 Subject: [PATCH 6/6] Allow skipping the test against s3 If RESTIC_TEST_S3_REPOSITORY is not available it means we're probably running on Travis for a third-party PR, so the access credentials for S3 are not exposed. --- run_integration_tests.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/run_integration_tests.go b/run_integration_tests.go index f820d49a9..eca5ed82f 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -155,8 +155,15 @@ func (env *TravisEnvironment) RunTests() error { "restic/backend/rest.TestBackendREST", "restic/backend/sftp.TestBackendSFTP", "restic/backend/s3.TestBackendMinio", - "restic/backend/s3.TestBackendS3", } + + // if the test s3 repository is available, make sure that the test is not skipped + if os.Getenv("RESTIC_TEST_S3_REPOSITORY") != "" { + ensureTests = append(ensureTests, "restic/backend/s3.TestBackendS3") + } else { + msg("S3 repository not available\n") + } + env.env["RESTIC_TEST_DISALLOW_SKIP"] = strings.Join(ensureTests, ",") if *runCrossCompile {