Merge pull request #960 from restic/travis-run-tests-against-s3

Enable CI tests against AWS S3

run_integration_tests.go

@@ -150,6 +150,22 @@ func (env *TravisEnvironment) RunTests() error {
 
 	env.env["GOPATH"] = cwd + ":" + filepath.Join(cwd, "vendor")
 
+	// ensure that the following tests cannot be silently skipped on Travis
+	ensureTests := []string{
+		"restic/backend/rest.TestBackendREST",
+		"restic/backend/sftp.TestBackendSFTP",
+		"restic/backend/s3.TestBackendMinio",
+	}
+
+	// if the test s3 repository is available, make sure that the test is not skipped
+	if os.Getenv("RESTIC_TEST_S3_REPOSITORY") != "" {
+		ensureTests = append(ensureTests, "restic/backend/s3.TestBackendS3")
+	} else {
+		msg("S3 repository not available\n")
+	}
+
+	env.env["RESTIC_TEST_DISALLOW_SKIP"] = strings.Join(ensureTests, ",")
+
 	if *runCrossCompile {
 		// compile for all target architectures with tags
 		for _, tags := range []string{"release", "debug"} {

src/restic/backend/rest/rest_test.go

@@ -60,7 +60,13 @@ func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
 	}
 }
 
-func TestBackend(t *testing.T) {
+func TestBackendREST(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 

src/restic/backend/s3/s3_test.go

@@ -97,6 +97,12 @@ func newCredentials(t testing.TB) (key, secret string) {
 }
 
 func TestBackendMinio(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/s3.TestBackendMinio")
+		}
+	}()
+
 	// try to find a minio binary
 	_, err := exec.LookPath("minio")
 	if err != nil {
@@ -169,7 +175,6 @@ func TestBackendMinio(t *testing.T) {
 				cfg.stopServer()
 			}
 			if cfg.removeTempdir != nil {
-				t.Logf("removeTempdir %v", config)
 				cfg.removeTempdir()
 			}
 			return nil
@@ -180,6 +185,12 @@ func TestBackendMinio(t *testing.T) {
 }
 
 func TestBackendS3(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/s3.TestBackendS3")
+		}
+	}()
+
 	vars := []string{
 		"RESTIC_TEST_S3_KEY",
 		"RESTIC_TEST_S3_SECRET",

src/restic/backend/sftp/sftp_test.go

@@ -29,7 +29,13 @@ func findSFTPServerBinary() string {
 
 var sftpServer = findSFTPServerBinary()
 
-func TestBackend(t *testing.T) {
+func TestBackendSFTP(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			SkipDisallowed(t, "restic/backend/sftp.TestBackendSFTP")
+		}
+	}()
+
 	if sftpServer == "" {
 		t.Skip("sftp server binary not found")
 	}

src/restic/backend/test/tests.go

@@ -277,7 +277,8 @@ func BackendTestLoad(t testing.TB, s *Suite) {
 
 type errorCloser struct {
 	io.Reader
-	t testing.TB
+	size int64
+	t    testing.TB
 }
 
 func (ec errorCloser) Close() error {
@@ -285,6 +286,10 @@ func (ec errorCloser) Close() error {
 	return errors.New("forbidden method close was called")
 }
 
+func (ec errorCloser) Size() int64 {
+	return ec.size
+}
+
 // BackendTestSave tests saving data in the backend.
 func BackendTestSave(t testing.TB, s *Suite) {
 	b := s.open(t)
@@ -354,7 +359,7 @@ func BackendTestSave(t testing.TB, s *Suite) {
 
 	// wrap the tempfile in an errorCloser, so we can detect if the backend
 	// closes the reader
-	err = b.Save(h, errorCloser{t: t, Reader: tmpfile})
+	err = b.Save(h, errorCloser{t: t, size: int64(length), Reader: tmpfile})
 	if err != nil {
 		t.Fatal(err)
 	}

src/restic/test/vars.go

@@ -3,19 +3,22 @@ package test
 import (
 	"fmt"
 	"os"
+	"strings"
+	"testing"
 )
 
 var (
-	TestPassword          = getStringVar("RESTIC_TEST_PASSWORD", "geheim")
-	TestCleanupTempDirs   = getBoolVar("RESTIC_TEST_CLEANUP", true)
-	TestTempDir           = getStringVar("RESTIC_TEST_TMPDIR", "")
-	RunIntegrationTest    = getBoolVar("RESTIC_TEST_INTEGRATION", true)
-	RunFuseTest           = getBoolVar("RESTIC_TEST_FUSE", true)
-	TestSFTPPath          = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec")
-	TestWalkerPath        = getStringVar("RESTIC_TEST_PATH", ".")
-	BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
-	TestS3Server          = getStringVar("RESTIC_TEST_S3_SERVER", "")
-	TestRESTServer        = getStringVar("RESTIC_TEST_REST_SERVER", "")
+	TestPassword                = getStringVar("RESTIC_TEST_PASSWORD", "geheim")
+	TestCleanupTempDirs         = getBoolVar("RESTIC_TEST_CLEANUP", true)
+	TestTempDir                 = getStringVar("RESTIC_TEST_TMPDIR", "")
+	RunIntegrationTest          = getBoolVar("RESTIC_TEST_INTEGRATION", true)
+	RunFuseTest                 = getBoolVar("RESTIC_TEST_FUSE", true)
+	TestSFTPPath                = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec")
+	TestWalkerPath              = getStringVar("RESTIC_TEST_PATH", ".")
+	BenchArchiveDirectory       = getStringVar("RESTIC_BENCH_DIR", ".")
+	TestS3Server                = getStringVar("RESTIC_TEST_S3_SERVER", "")
+	TestRESTServer              = getStringVar("RESTIC_TEST_REST_SERVER", "")
+	TestIntegrationDisallowSkip = getStringVar("RESTIC_TEST_DISALLOW_SKIP", "")
 )
 
 func getStringVar(name, defaultValue string) string {
@@ -40,3 +43,15 @@ func getBoolVar(name string, defaultValue bool) bool {
 
 	return defaultValue
 }
+
+// SkipDisallowed fails the test if it needs to run. The environment
+// variable RESTIC_TEST_DISALLOW_SKIP contains a comma-separated list of test
+// names that must be run. If name is in this list, the test is marked as
+// failed.
+func SkipDisallowed(t testing.TB, name string) {
+	for _, s := range strings.Split(TestIntegrationDisallowSkip, ",") {
+		if s == name {
+			t.Fatalf("test %v is in list of tests that need to run ($RESTIC_TEST_DISALLOW_SKIP)", name)
+		}
+	}
+}

vendor/src/github.com/minio/minio-go/api-put-bucket.go

@@ -89,7 +89,7 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) {
 		if resp.StatusCode != http.StatusOK {
 			err := httpRespToErrorResponse(resp, bucketName, "")
 			errResp := ToErrorResponse(err)
-			if errResp.Code == "InvalidRegion" && errResp.Region != "" {
+			if resp.StatusCode == http.StatusBadRequest && errResp.Region != "" {
 				// Fetch bucket region found in headers
 				// of S3 error response, attempt bucket
 				// create again.

vendor/src/github.com/minio/minio-go/api-put-object-common.go

@@ -157,11 +157,13 @@ func hashCopyN(hashAlgorithms map[string]hash.Hash, hashSums map[string][]byte,
 			return 0, err
 		}
 	}
-
-	for k, v := range hashAlgorithms {
-		hashSums[k] = v.Sum(nil)
+	if err == nil && size == partSize {
+		for k, v := range hashAlgorithms {
+			hashSums[k] = v.Sum(nil)
+		}
+		return size, nil
 	}
-	return size, err
+	return 0, ErrUnexpectedEOF(size, partSize, "", "")
 }
 
 // getUploadID - fetch upload id if already present for an object name

vendor/src/github.com/minio/minio-go/api.go

@@ -553,7 +553,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt
 		// Bucket region if set in error response and the error
 		// code dictates invalid region, we can retry the request
 		// with the new region.
-		if errResponse.Region != "" && res.StatusCode == http.StatusBadRequest {
+		if res.StatusCode == http.StatusBadRequest && errResponse.Region != "" {
 			c.bucketLocCache.Set(metadata.bucketName, errResponse.Region)
 			continue // Retry.
 		}