diff --git a/doc/tutorial_aws_s3.rst b/doc/tutorial_aws_s3.rst index 999373e04..e1f89c969 100644 --- a/doc/tutorial_aws_s3.rst +++ b/doc/tutorial_aws_s3.rst @@ -1,12 +1,12 @@ -Setting up Restic with Amazon S3 +Setting up restic with Amazon S3 ================================ Preface ------- -This tutorial will show you how to use Restic with AWS S3. It will show you how +This tutorial will show you how to use restic with AWS S3. It will show you how to navigate the AWS web interface, create an S3 bucket, create a user with -access to only this bucket, and finally how to connect Restic to this bucket. +access to only this bucket, and finally how to connect restic to this bucket. Prerequisites ------------- @@ -95,8 +95,8 @@ AWS through the ``restic`` program and not through the web interface. Therefore, :alt: Choose User Name and Access Type During the next step, permissions can be assigned to the new user. To use this -user with Restic, it only needs access to the ``restic-demo`` bucket. Select -"Attach exiting policies directly", which will bring up a list of pre-defined +user with restic, it only needs access to the ``restic-demo`` bucket. Select +"Attach existing policies directly", which will bring up a list of pre-defined policies below. Afterwards, click the "Create policy" button to create a custom policy: @@ -111,17 +111,17 @@ Generator" will be used to generate a policy file using a web interface: :alt: Create a New Policy After invoking the policy generator, you will be presented with a user -interface to generate individual permission statements. For Restic to work, two +interface to generate individual permission statements. For restic to work, two such statements must be created. The first statement is set up as follows: .. code:: Effect: Allow - Service: S3 + Service: Amazon S3 Actions: DeleteObject, GetObject, PutObject Resource: arn:aws:s3:::restic-demo/* -This statement allows Restic to create, read and delete objects inside the S3 +This statement allows restic to create, read and delete objects inside the S3 bucket named ``restic-demo``. Adjust the bucket's name to the name of the bucket you created earlier. Using the "Add Statement" button, this statement can be saved. Now a second statement is created: @@ -129,13 +129,13 @@ saved. Now a second statement is created: .. code:: Effect: Allow - Service: S3 + Service: Amazon S3 Actions: ListBucket Resource: arn:aws:s3:::restic-demo Again, substitute ``restic-demo`` with the actual name of your bucket. Note that, unlike before, there is no ``/*`` after the bucket name. This statement allows -Restic to list the objects stored in the ``restic-demo`` bucket. Again, use "Add +restic to list the objects stored in the ``restic-demo`` bucket. Again, use "Add Statement" to save this statement. The policy creator interface should now look as follows: @@ -176,7 +176,7 @@ You have now completed the configuration in AWS. Feel free to close your web browser now. -Initializing the Restic repository +Initializing the restic repository ---------------------------------- Open a terminal and make sure you have the ``restic`` binary ready. First, choose @@ -189,7 +189,7 @@ this purpose: I9n7G7G0ZpDWA3GOcJbIuwQCGvGUBkU5 Note this password somewhere safe along with your AWS credentials. Next, the -configuration of Restic will be placed into environment variables. This will +configuration of restic will be placed into environment variables. This will include sensitive information, such as your AWS secret and repository password. Therefore, make sure the next commands **do not** end up in your shell's history file. Adjust the contents of the environment variables to fit your @@ -204,7 +204,7 @@ bucket's name and your user's API credentials. $ export RESTIC_PASSWORD="I9n7G7G0ZpDWA3GOcJbIuwQCGvGUBkU5" -After the environment is set up, Restic may be called to initialize the +After the environment is set up, restic may be called to initialize the repository: @@ -217,7 +217,7 @@ repository: the repository. Losing your password means that your data is irrecoverably lost. -Restic is now ready to be used with AWS S3. Try to create a backup: +restic is now ready to be used with AWS S3. Try to create a backup: .. code-block:: console diff --git a/run_integration_tests.go b/run_integration_tests.go index 39c2be53f..00fe598fc 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -9,7 +9,6 @@ import ( "flag" "fmt" "io" - "io/ioutil" "net/http" "os" "os/exec" @@ -25,21 +24,6 @@ var ForbiddenImports = map[string]bool{ } var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests") -var minioServer = flag.String("minio", "", "path to the minio server binary") -var restServer = flag.String("rest", "", "path to the rest-server binary") -var debug = flag.Bool("debug", false, "output debug messages") - -var minioServerEnv = map[string]string{ - "MINIO_ACCESS_KEY": "KEBIYDZ87HCIH5D17YCN", - "MINIO_SECRET_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", -} - -var minioEnv = map[string]string{ - "RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000", - "RESTIC_TEST_REST_SERVER": "http://127.0.0.1:8000", - "AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN", - "AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", -} func init() { flag.Parse() @@ -55,32 +39,11 @@ type CIEnvironment interface { // TravisEnvironment is the environment in which Travis tests run. type TravisEnvironment struct { goxOSArch []string - - minio string - minioSrv *Background - minioTempdir string - - rest string - restSrv *Background - restTempdir string - - env map[string]string + env map[string]string } func (env *TravisEnvironment) getMinio() error { - if *minioServer != "" { - msg("using minio server at %q\n", *minioServer) - env.minio = *minioServer - return nil - } - - if *restServer != "" { - msg("using REST server at %q\n", *restServer) - env.rest = *restServer - return nil - } - - tempfile, err := ioutil.TempFile("", "minio-server-") + tempfile, err := os.Create(filepath.Join(os.Getenv("GOPATH"), "bin", "minio")) if err != nil { return fmt.Errorf("create tempfile for minio download failed: %v\n", err) } @@ -115,64 +78,6 @@ func (env *TravisEnvironment) getMinio() error { } msg("downloaded minio server to %v\n", tempfile.Name()) - env.minio = tempfile.Name() - return nil -} - -func (env *TravisEnvironment) runMinio() error { - if env.minio == "" { - return nil - } - - // start minio server - msg("starting minio server at %s", env.minio) - - dir, err := ioutil.TempDir("", "minio-root") - if err != nil { - return fmt.Errorf("TempDir: %v", err) - } - - env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio, - "server", - "--address", "127.0.0.1:9000", - dir) - if err != nil { - return fmt.Errorf("error running minio server: %v", err) - } - - // go func() { - // time.Sleep(300 * time.Millisecond) - // env.minioSrv.Cmd.Process.Kill() - // }() - - for k, v := range minioEnv { - env.env[k] = v - } - - env.minioTempdir = dir - return nil -} - -func (env *TravisEnvironment) runRESTServer() error { - if env.rest == "" { - return nil - } - - // start rest server - msg("starting rest server at %s", env.rest) - - dir, err := ioutil.TempDir("", "rest-server-root") - if err != nil { - return fmt.Errorf("TempDir: %v", err) - } - - env.restSrv, err = StartBackgroundCommand(map[string]string{}, env.rest, - "--path", dir) - if err != nil { - return fmt.Errorf("error running rest server: %v", err) - } - - env.restTempdir = dir return nil } @@ -186,10 +91,7 @@ func (env *TravisEnvironment) Prepare() error { "golang.org/x/tools/cmd/cover", "github.com/pierrre/gotestcover", "github.com/NebulousLabs/glyphcheck", - } - - if env.rest == "" { - pkgs = append(pkgs, "github.com/restic/rest-server") + "github.com/restic/rest-server", } for _, pkg := range pkgs { @@ -199,19 +101,9 @@ func (env *TravisEnvironment) Prepare() error { } } - if env.rest == "" { - env.rest = filepath.Join(os.Getenv("GOPATH"), "bin", "rest-server") - } - if err := env.getMinio(); err != nil { return err } - if err := env.runMinio(); err != nil { - return err - } - if err := env.runRESTServer(); err != nil { - return err - } if *runCrossCompile { // only test cross compilation on linux with Travis @@ -240,110 +132,9 @@ func (env *TravisEnvironment) Prepare() error { // Teardown stops backend services and cleans the environment again. func (env *TravisEnvironment) Teardown() error { msg("run travis teardown\n") - if env.minioSrv != nil { - msg("stopping minio server\n") - - if env.minioSrv.Cmd.ProcessState == nil { - err := env.minioSrv.Cmd.Process.Kill() - if err != nil { - fmt.Fprintf(os.Stderr, "error killing minio server process: %v", err) - } - } else { - result := <-env.minioSrv.Result - if result.Error != nil { - msg("minio server returned error: %v\n", result.Error) - msg("stdout: %s\n", result.Stdout) - msg("stderr: %s\n", result.Stderr) - } - } - - err := os.RemoveAll(env.minioTempdir) - if err != nil { - msg("error removing minio tempdir %v: %v\n", env.minioTempdir, err) - } - } - - if env.restSrv != nil { - msg("stopping rest-server\n") - - if env.restSrv.Cmd.ProcessState == nil { - err := env.restSrv.Cmd.Process.Kill() - if err != nil { - fmt.Fprintf(os.Stderr, "error killing rest-server process: %v", err) - } - } else { - result := <-env.restSrv.Result - if result.Error != nil { - msg("rest-server returned error: %v\n", result.Error) - msg("stdout: %s\n", result.Stdout) - msg("stderr: %s\n", result.Stderr) - } - } - - err := os.RemoveAll(env.restTempdir) - if err != nil { - msg("error removing rest-server tempdir %v: %v\n", env.restTempdir, err) - } - } - return nil } -// Background is a program running in the background. -type Background struct { - Cmd *exec.Cmd - Result chan Result -} - -// Result is the result of a program that ran in the background. -type Result struct { - Stdout, Stderr string - Error error -} - -// StartBackgroundCommand runs a program in the background. -func StartBackgroundCommand(env map[string]string, cmd string, args ...string) (*Background, error) { - msg("running background command %v %v\n", cmd, args) - b := Background{ - Result: make(chan Result, 1), - } - - stdout := bytes.NewBuffer(nil) - stderr := bytes.NewBuffer(nil) - - c := exec.Command(cmd, args...) - c.Stdout = stdout - c.Stderr = stderr - - if *debug { - c.Stdout = io.MultiWriter(c.Stdout, os.Stdout) - c.Stderr = io.MultiWriter(c.Stderr, os.Stderr) - } - c.Env = updateEnv(os.Environ(), env) - - b.Cmd = c - - err := c.Start() - if err != nil { - msg("error starting background job %v: %v\n", cmd, err) - return nil, err - } - - go func() { - err := b.Cmd.Wait() - msg("background job %v returned: %v\n", cmd, err) - msg("stdout: %s\n", stdout.Bytes()) - msg("stderr: %s\n", stderr.Bytes()) - b.Result <- Result{ - Stdout: string(stdout.Bytes()), - Stderr: string(stderr.Bytes()), - Error: err, - } - }() - - return &b, nil -} - // RunTests starts the tests for Travis. func (env *TravisEnvironment) RunTests() error { // do not run fuse tests on darwin @@ -436,16 +227,16 @@ func (env *AppveyorEnvironment) Teardown() error { // findGoFiles returns a list of go source code file names below dir. func findGoFiles(dir string) (list []string, err error) { err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error { - if filepath.Base(name) == "vendor" { + relpath, err := filepath.Rel(dir, name) + if err != nil { + return err + } + + if relpath == "vendor" || relpath == "pkg" { return filepath.SkipDir } - if filepath.Ext(name) == ".go" { - relpath, err := filepath.Rel(dir, name) - if err != nil { - return err - } - + if filepath.Ext(relpath) == ".go" { list = append(list, relpath) } diff --git a/src/cmds/restic/main.go b/src/cmds/restic/main.go index 96a508da9..b3877c5b1 100644 --- a/src/cmds/restic/main.go +++ b/src/cmds/restic/main.go @@ -1,7 +1,10 @@ package main import ( + "bufio" + "bytes" "fmt" + "log" "os" "restic" "restic/debug" @@ -44,6 +47,14 @@ directories in an encrypted repository stored on different backends. }, } +var logBuffer = bytes.NewBuffer(nil) + +func init() { + // install custom global logger into a buffer, if an error occurs + // we can show the logs + log.SetOutput(logBuffer) +} + func main() { debug.Log("main %#v", os.Args) err := cmdRoot.Execute() @@ -55,6 +66,14 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) case err != nil: fmt.Fprintf(os.Stderr, "%+v\n", err) + + if logBuffer.Len() > 0 { + fmt.Fprintf(os.Stderr, "also, the following messages were logged by a library:\n") + sc := bufio.NewScanner(logBuffer) + for sc.Scan() { + fmt.Fprintln(os.Stderr, sc.Text()) + } + } } var exitCode int diff --git a/src/restic/backend/http_transport.go b/src/restic/backend/http_transport.go new file mode 100644 index 000000000..1f65ac7c3 --- /dev/null +++ b/src/restic/backend/http_transport.go @@ -0,0 +1,28 @@ +package backend + +import ( + "net" + "net/http" + "restic/debug" + "time" +) + +// Transport returns a new http.RoundTripper with default settings applied. +func Transport() http.RoundTripper { + // copied from net/http + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + // wrap in the debug round tripper + return debug.RoundTripper(tr) +} diff --git a/src/restic/backend/local/backend_test.go b/src/restic/backend/local/backend_test.go deleted file mode 100644 index 8607f01b7..000000000 --- a/src/restic/backend/local/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package local_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestLocalBackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestLocalBackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestLocalBackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestLocalBackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestLocalBackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestLocalBackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestLocalBackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestLocalBackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestLocalBackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestLocalBackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestLocalBackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/local/local_test.go b/src/restic/backend/local/local_test.go index 7e7440561..ad0d626cb 100644 --- a/src/restic/backend/local/local_test.go +++ b/src/restic/backend/local/local_test.go @@ -1,59 +1,55 @@ package local_test import ( - "fmt" "io/ioutil" - "os" "restic" + "testing" "restic/backend/local" "restic/backend/test" + . "restic/test" ) -var tempBackendDir string +func TestBackend(t *testing.T) { + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-") + if err != nil { + t.Fatal(err) + } -//go:generate go run ../test/generate_backend_tests.go + t.Logf("create new backend at %v", dir) -func createTempdir() error { - if tempBackendDir != "" { - return nil - } + cfg := local.Config{ + Path: dir, + } + return cfg, nil + }, - tempdir, err := ioutil.TempDir("", "restic-local-test-") - if err != nil { - return err - } + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(local.Config) + return local.Create(cfg) + }, - fmt.Printf("created new test backend at %v\n", tempdir) - tempBackendDir = tempdir - return nil -} + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(local.Config) + return local.Open(cfg) + }, -func init() { - test.CreateFn = func() (restic.Backend, error) { - err := createTempdir() - if err != nil { - return nil, err - } - return local.Create(local.Config{Path: tempBackendDir}) - } + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + cfg := config.(local.Config) + if !TestCleanupTempDirs { + t.Logf("leaving test backend dir at %v", cfg.Path) + } - test.OpenFn = func() (restic.Backend, error) { - err := createTempdir() - if err != nil { - return nil, err - } - return local.Open(local.Config{Path: tempBackendDir}) - } - - test.CleanupFn = func() error { - if tempBackendDir == "" { + RemoveAll(t, cfg.Path) return nil - } - - fmt.Printf("removing test backend at %v\n", tempBackendDir) - err := os.RemoveAll(tempBackendDir) - tempBackendDir = "" - return err + }, } + + suite.RunTests(t) } diff --git a/src/restic/backend/mem/backend_test.go b/src/restic/backend/mem/backend_test.go deleted file mode 100644 index 13e95f115..000000000 --- a/src/restic/backend/mem/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package mem_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestMemBackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestMemBackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestMemBackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestMemBackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestMemBackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestMemBackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestMemBackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestMemBackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestMemBackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestMemBackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestMemBackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/mem/mem_backend_test.go b/src/restic/backend/mem/mem_backend_test.go index 75b65f4c7..2f39787a9 100644 --- a/src/restic/backend/mem/mem_backend_test.go +++ b/src/restic/backend/mem/mem_backend_test.go @@ -2,6 +2,7 @@ package mem_test import ( "restic" + "testing" "restic/errors" @@ -9,31 +10,50 @@ import ( "restic/backend/test" ) -var be restic.Backend - -//go:generate go run ../test/generate_backend_tests.go - -func init() { - test.CreateFn = func() (restic.Backend, error) { - if be != nil { - return nil, errors.New("temporary memory backend dir already exists") - } - - be = mem.New() - - return be, nil - } - - test.OpenFn = func() (restic.Backend, error) { - if be == nil { - return nil, errors.New("repository not initialized") - } - - return be, nil - } - - test.CleanupFn = func() error { - be = nil - return nil - } +type memConfig struct { + be restic.Backend +} + +func TestSuiteBackendMem(t *testing.T) { + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + return &memConfig{}, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(cfg interface{}) (restic.Backend, error) { + c := cfg.(*memConfig) + if c.be != nil { + ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile}) + if err != nil { + return nil, err + } + + if ok { + return nil, errors.New("config already exists") + } + } + + c.be = mem.New() + return c.be, nil + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(cfg interface{}) (restic.Backend, error) { + c := cfg.(*memConfig) + if c.be == nil { + c.be = mem.New() + } + return c.be, nil + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(cfg interface{}) error { + // no cleanup needed + return nil + }, + } + + suite.RunTests(t) } diff --git a/src/restic/backend/rest/backend_test.go b/src/restic/backend/rest/backend_test.go deleted file mode 100644 index 4274bfcb1..000000000 --- a/src/restic/backend/rest/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package rest_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestRestBackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestRestBackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestRestBackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestRestBackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestRestBackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestRestBackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestRestBackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestRestBackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestRestBackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestRestBackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestRestBackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index 773fad6bd..5a0e6226b 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -35,8 +35,8 @@ func Open(cfg Config) (restic.Backend, error) { for i := 0; i < connLimit; i++ { connChan <- struct{}{} } - tr := &http.Transport{MaxIdleConnsPerHost: connLimit} - client := http.Client{Transport: tr} + + client := http.Client{Transport: backend.Transport()} // use url without trailing slash for layout url := cfg.URL.String() diff --git a/src/restic/backend/rest/rest_test.go b/src/restic/backend/rest/rest_test.go index f6f7676ef..d378a364f 100644 --- a/src/restic/backend/rest/rest_test.go +++ b/src/restic/backend/rest/rest_test.go @@ -1,39 +1,113 @@ package rest_test import ( - "fmt" + "context" + "io/ioutil" + "net" "net/url" "os" + "os/exec" "restic" + "testing" + "time" "restic/backend/rest" "restic/backend/test" . "restic/test" ) -//go:generate go run ../test/generate_backend_tests.go - -func init() { - if TestRESTServer == "" { - SkipMessage = "REST test server not available" - return - } - - url, err := url.Parse(TestRESTServer) +func runRESTServer(ctx context.Context, t testing.TB, dir string) func() { + srv, err := exec.LookPath("rest-server") if err != nil { - fmt.Fprintf(os.Stderr, "invalid url: %v\n", err) - return + t.Skip(err) } - cfg := rest.Config{ - URL: url, + cmd := exec.CommandContext(ctx, srv, "--path", dir) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stdout + if err := cmd.Start(); err != nil { + t.Fatal(err) } - test.CreateFn = func() (restic.Backend, error) { - return rest.Create(cfg) + // wait until the TCP port is reachable + var success bool + for i := 0; i < 10; i++ { + time.Sleep(200 * time.Millisecond) + + c, err := net.Dial("tcp", "localhost:8000") + if err != nil { + continue + } + + success = true + if err := c.Close(); err != nil { + t.Fatal(err) + } } - test.OpenFn = func() (restic.Backend, error) { - return rest.Open(cfg) + if !success { + t.Fatal("unable to connect to rest server") + return nil + } + + return func() { + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + + // ignore errors, we've killed the process + _ = cmd.Wait() } } + +func TestBackend(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + dir, cleanup := TempDir(t) + defer cleanup() + + cleanup = runRESTServer(ctx, t, dir) + defer cleanup() + + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + dir, err := ioutil.TempDir(TestTempDir, "restic-test-rest-") + if err != nil { + t.Fatal(err) + } + + t.Logf("create new backend at %v", dir) + + url, err := url.Parse("http://localhost:8000/restic-test") + if err != nil { + t.Fatal(err) + } + + cfg := rest.Config{ + URL: url, + } + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(rest.Config) + return rest.Create(cfg) + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(rest.Config) + return rest.Open(cfg) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + return nil + }, + } + + suite.RunTests(t) +} diff --git a/src/restic/backend/s3/backend_test.go b/src/restic/backend/s3/backend_test.go deleted file mode 100644 index 82eca2631..000000000 --- a/src/restic/backend/s3/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package s3_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestS3BackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestS3BackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestS3BackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestS3BackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestS3BackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestS3BackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestS3BackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestS3BackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestS3BackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestS3BackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestS3BackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 2f6db84f5..f16b3c594 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -3,7 +3,6 @@ package s3 import ( "bytes" "io" - "net/http" "path" "restic" "strings" @@ -48,8 +47,7 @@ func Open(cfg Config) (restic.Backend, error) { Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join}, } - tr := &http.Transport{MaxIdleConnsPerHost: connLimit} - client.SetCustomTransport(tr) + client.SetCustomTransport(backend.Transport()) be.createConnections() diff --git a/src/restic/backend/s3/s3_test.go b/src/restic/backend/s3/s3_test.go index 3bee8c8cd..f50bfd680 100644 --- a/src/restic/backend/s3/s3_test.go +++ b/src/restic/backend/s3/s3_test.go @@ -1,73 +1,260 @@ package s3_test import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" "fmt" - "net/url" + "io" + "net" "os" + "os/exec" + "path/filepath" "restic" - - "restic/errors" + "testing" + "time" "restic/backend/s3" "restic/backend/test" . "restic/test" ) -//go:generate go run ../test/generate_backend_tests.go - -func init() { - if TestS3Server == "" { - SkipMessage = "s3 test server not available" - return - } - - url, err := url.Parse(TestS3Server) +func mkdir(t testing.TB, dir string) { + err := os.MkdirAll(dir, 0700) if err != nil { - fmt.Fprintf(os.Stderr, "invalid url: %v\n", err) + t.Fatal(err) + } +} + +func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func() { + mkdir(t, filepath.Join(dir, "config")) + mkdir(t, filepath.Join(dir, "root")) + + cmd := exec.CommandContext(ctx, "minio", + "server", + "--address", "127.0.0.1:9000", + "--config-dir", filepath.Join(dir, "config"), + filepath.Join(dir, "root")) + cmd.Env = append(os.Environ(), + "MINIO_ACCESS_KEY="+key, + "MINIO_SECRET_KEY="+secret, + ) + cmd.Stderr = os.Stderr + + err := cmd.Start() + if err != nil { + t.Fatal(err) + } + + // wait until the TCP port is reachable + var success bool + for i := 0; i < 10; i++ { + time.Sleep(200 * time.Millisecond) + + c, err := net.Dial("tcp", "localhost:9000") + if err != nil { + continue + } + + success = true + if err := c.Close(); err != nil { + t.Fatal(err) + } + } + + if !success { + t.Fatal("unable to connect to minio server") + return nil + } + + return func() { + err = cmd.Process.Kill() + if err != nil { + t.Fatal(err) + } + + // ignore errors, we've killed the process + _ = cmd.Wait() + } +} + +func newCredentials(t testing.TB) (key, secret string) { + buf := make([]byte, 10) + _, err := io.ReadFull(rand.Reader, buf) + if err != nil { + t.Fatal(err) + } + key = hex.EncodeToString(buf) + + _, err = io.ReadFull(rand.Reader, buf) + if err != nil { + t.Fatal(err) + } + secret = hex.EncodeToString(buf) + + return key, secret +} + +func TestBackendMinio(t *testing.T) { + // try to find a minio binary + _, err := exec.LookPath("minio") + if err != nil { + t.Skip(err) return } - cfg := s3.Config{ - Endpoint: url.Host, - Bucket: "restictestbucket", - KeyID: os.Getenv("AWS_ACCESS_KEY_ID"), - Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"), + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + type Config struct { + s3.Config + + tempdir string + removeTempdir func() + stopServer func() } - if url.Scheme == "http" { - cfg.UseHTTP = true + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + cfg := Config{} + + cfg.tempdir, cfg.removeTempdir = TempDir(t) + key, secret := newCredentials(t) + cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret) + + cfg.Config = s3.Config{ + Endpoint: "localhost:9000", + Bucket: "restictestbucket", + Prefix: fmt.Sprintf("test-%d", time.Now().UnixNano()), + UseHTTP: true, + KeyID: key, + Secret: secret, + } + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(Config) + + be, err := s3.Open(cfg.Config) + if err != nil { + return nil, err + } + + exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + if err != nil { + return nil, err + } + + if exists { + return nil, errors.New("config already exists") + } + + return be, nil + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(Config) + return s3.Open(cfg.Config) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + cfg := config.(Config) + if cfg.stopServer != nil { + cfg.stopServer() + } + if cfg.removeTempdir != nil { + t.Logf("removeTempdir %v", config) + cfg.removeTempdir() + } + return nil + }, } - test.CreateFn = func() (restic.Backend, error) { - be, err := s3.Open(cfg) - if err != nil { - return nil, err - } - - exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) - if err != nil { - return nil, err - } - - if exists { - return nil, errors.New("config already exists") - } - - return be, nil - } - - test.OpenFn = func() (restic.Backend, error) { - return s3.Open(cfg) - } - - // test.CleanupFn = func() error { - // if tempBackendDir == "" { - // return nil - // } - - // fmt.Printf("removing test backend at %v\n", tempBackendDir) - // err := os.RemoveAll(tempBackendDir) - // tempBackendDir = "" - // return err - // } + suite.RunTests(t) +} + +func TestBackendS3(t *testing.T) { + vars := []string{ + "RESTIC_TEST_S3_KEY", + "RESTIC_TEST_S3_SECRET", + "RESTIC_TEST_S3_REPOSITORY", + } + + for _, v := range vars { + if os.Getenv(v) == "" { + t.Skipf("environment variable %v not set", v) + return + } + } + + suite := test.Suite{ + // do not use excessive data + MinimalData: true, + + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY")) + if err != nil { + return nil, err + } + + cfg := s3cfg.(s3.Config) + cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY") + cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET") + cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano()) + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(s3.Config) + + be, err := s3.Open(cfg) + if err != nil { + return nil, err + } + + exists, err := be.Test(restic.Handle{Type: restic.ConfigFile}) + if err != nil { + return nil, err + } + + if exists { + return nil, errors.New("config already exists") + } + + return be, nil + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(s3.Config) + return s3.Open(cfg) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + cfg := config.(s3.Config) + + be, err := s3.Open(cfg) + if err != nil { + return err + } + + if err := be.(restic.Deleter).Delete(); err != nil { + return err + } + + return nil + }, + } + + t.Logf("run tests") + suite.RunTests(t) } diff --git a/src/restic/backend/sftp/backend_test.go b/src/restic/backend/sftp/backend_test.go deleted file mode 100644 index a812f8cd0..000000000 --- a/src/restic/backend/sftp/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package sftp_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestSftpBackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestSftpBackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestSftpBackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestSftpBackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestSftpBackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestSftpBackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestSftpBackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestSftpBackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestSftpBackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestSftpBackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestSftpBackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/sftp/layout_test.go b/src/restic/backend/sftp/layout_test.go index 0159976fe..856bd5ce4 100644 --- a/src/restic/backend/sftp/layout_test.go +++ b/src/restic/backend/sftp/layout_test.go @@ -10,7 +10,7 @@ import ( ) func TestLayout(t *testing.T) { - if sftpserver == "" { + if sftpServer == "" { t.Skip("sftp server binary not available") } @@ -46,7 +46,7 @@ func TestLayout(t *testing.T) { repo := filepath.Join(path, "repo") be, err := sftp.Open(sftp.Config{ - Command: fmt.Sprintf("%q -e", sftpserver), + Command: fmt.Sprintf("%q -e", sftpServer), Path: repo, Layout: test.layout, }) diff --git a/src/restic/backend/sftp/sftp.go b/src/restic/backend/sftp/sftp.go index 8f56e23e4..3eedfa76d 100644 --- a/src/restic/backend/sftp/sftp.go +++ b/src/restic/backend/sftp/sftp.go @@ -294,7 +294,7 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) { // save data _, err = io.Copy(f, rd) if err != nil { - f.Close() + _ = f.Close() return errors.Wrap(err, "Write") } diff --git a/src/restic/backend/sftp/sftp_backend_test.go b/src/restic/backend/sftp/sftp_backend_test.go deleted file mode 100644 index 1834beee8..000000000 --- a/src/restic/backend/sftp/sftp_backend_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package sftp_test - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "restic" - "strings" - - "restic/errors" - - "restic/backend/sftp" - "restic/backend/test" - - . "restic/test" -) - -var tempBackendDir string - -//go:generate go run ../test/generate_backend_tests.go - -func createTempdir() error { - if tempBackendDir != "" { - return nil - } - - tempdir, err := ioutil.TempDir("", "restic-local-test-") - if err != nil { - return err - } - - tempBackendDir = tempdir - return nil -} - -func findSFTPServerBinary() string { - for _, dir := range strings.Split(TestSFTPPath, ":") { - testpath := filepath.Join(dir, "sftp-server") - _, err := os.Stat(testpath) - if !os.IsNotExist(errors.Cause(err)) { - return testpath - } - } - - return "" -} - -var sftpserver = findSFTPServerBinary() - -func init() { - if sftpserver == "" { - SkipMessage = "sftp server binary not found, skipping tests" - return - } - - cfg := sftp.Config{ - Command: fmt.Sprintf("%q -e", sftpserver), - } - - test.CreateFn = func() (restic.Backend, error) { - err := createTempdir() - if err != nil { - return nil, err - } - - cfg.Path = tempBackendDir - - return sftp.Create(cfg) - } - - test.OpenFn = func() (restic.Backend, error) { - err := createTempdir() - if err != nil { - return nil, err - } - - cfg.Path = tempBackendDir - - return sftp.Open(cfg) - } - - test.CleanupFn = func() error { - if tempBackendDir == "" { - return nil - } - - err := os.RemoveAll(tempBackendDir) - tempBackendDir = "" - return err - } -} diff --git a/src/restic/backend/sftp/sftp_test.go b/src/restic/backend/sftp/sftp_test.go new file mode 100644 index 000000000..dd9728676 --- /dev/null +++ b/src/restic/backend/sftp/sftp_test.go @@ -0,0 +1,79 @@ +package sftp_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "restic" + "restic/backend/sftp" + "restic/backend/test" + "restic/errors" + "strings" + "testing" + + . "restic/test" +) + +func findSFTPServerBinary() string { + for _, dir := range strings.Split(TestSFTPPath, ":") { + testpath := filepath.Join(dir, "sftp-server") + _, err := os.Stat(testpath) + if !os.IsNotExist(errors.Cause(err)) { + return testpath + } + } + + return "" +} + +var sftpServer = findSFTPServerBinary() + +func TestBackend(t *testing.T) { + if sftpServer == "" { + t.Skip("sftp server binary not found") + } + + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + dir, err := ioutil.TempDir(TestTempDir, "restic-test-sftp-") + if err != nil { + t.Fatal(err) + } + + t.Logf("create new backend at %v", dir) + + cfg := sftp.Config{ + Path: dir, + Command: fmt.Sprintf("%q -e", sftpServer), + } + return cfg, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(config interface{}) (restic.Backend, error) { + cfg := config.(sftp.Config) + return sftp.Create(cfg) + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(config interface{}) (restic.Backend, error) { + cfg := config.(sftp.Config) + return sftp.Open(cfg) + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(config interface{}) error { + cfg := config.(sftp.Config) + if !TestCleanupTempDirs { + t.Logf("leaving test backend dir at %v", cfg.Path) + } + + RemoveAll(t, cfg.Path) + return nil + }, + } + + suite.RunTests(t) +} diff --git a/src/restic/backend/test/backend_test.go b/src/restic/backend/test/backend_test.go deleted file mode 100644 index b495ce663..000000000 --- a/src/restic/backend/test/backend_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// DO NOT EDIT, AUTOMATICALLY GENERATED -package test_test - -import ( - "testing" - - "restic/backend/test" -) - -var SkipMessage string - -func TestTestBackendCreate(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreate(t) -} - -func TestTestBackendOpen(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestOpen(t) -} - -func TestTestBackendCreateWithConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCreateWithConfig(t) -} - -func TestTestBackendLocation(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLocation(t) -} - -func TestTestBackendConfig(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestConfig(t) -} - -func TestTestBackendLoad(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestLoad(t) -} - -func TestTestBackendSave(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSave(t) -} - -func TestTestBackendSaveFilenames(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestSaveFilenames(t) -} - -func TestTestBackendBackend(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestBackend(t) -} - -func TestTestBackendDelete(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestDelete(t) -} - -func TestTestBackendCleanup(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestCleanup(t) -} diff --git a/src/restic/backend/test/funcs.go b/src/restic/backend/test/funcs.go new file mode 100644 index 000000000..c27d23f3b --- /dev/null +++ b/src/restic/backend/test/funcs.go @@ -0,0 +1,21 @@ +// DO NOT EDIT, AUTOMATICALLY GENERATED + +package test + +import ( + "testing" +) + +var testFunctions = []struct { + Name string + Fn func(t testing.TB, suite *Suite) +}{ + {"CreateWithConfig", BackendTestCreateWithConfig}, + {"Location", BackendTestLocation}, + {"Config", BackendTestConfig}, + {"Load", BackendTestLoad}, + {"Save", BackendTestSave}, + {"SaveFilenames", BackendTestSaveFilenames}, + {"Backend", BackendTestBackend}, + {"Delete", BackendTestDelete}, +} diff --git a/src/restic/backend/test/generate_backend_tests.go b/src/restic/backend/test/generate_test_list.go similarity index 72% rename from src/restic/backend/test/generate_backend_tests.go rename to src/restic/backend/test/generate_test_list.go index 4d1e9b478..30b310e73 100644 --- a/src/restic/backend/test/generate_backend_tests.go +++ b/src/restic/backend/test/generate_test_list.go @@ -18,35 +18,31 @@ import ( ) var data struct { - Package string - PackagePrefix string - Funcs []string + Package string + Funcs []string } var testTemplate = ` // DO NOT EDIT, AUTOMATICALLY GENERATED + package {{ .Package }} import ( "testing" - - "restic/backend/test" ) -var SkipMessage string - -{{ $prefix := .PackagePrefix }} -{{ range $f := .Funcs }} -func Test{{ $prefix }}{{ $f }}(t *testing.T){ - if SkipMessage != "" { t.Skip(SkipMessage) } - test.Test{{ $f }}(t) -} - +var testFunctions = []struct { + Name string + Fn func(t testing.TB, suite *Suite) +}{ +{{ range $f := .Funcs -}} + {"{{ $f }}", BackendTest{{ $f }},}, {{ end }} +} ` -var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in") -var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to") +var testFile = flag.String("testfile", "tests.go", "file to search test functions in") +var outputFile = flag.String("output", "funcs.go", "output file to write generated code to") var packageName = flag.String("package", "", "the package name to use") var prefix = flag.String("prefix", "", "test function prefix") var quiet = flag.Bool("quiet", false, "be quiet") @@ -60,7 +56,7 @@ func errx(err error) { os.Exit(1) } -var funcRegex = regexp.MustCompile(`^func\s+Test(.+)\s*\(`) +var funcRegex = regexp.MustCompile(`^func\s+BackendTest(.+)\s*\(`) func findTestFunctions() (funcs []string) { f, err := os.Open(*testFile) @@ -123,12 +119,7 @@ func main() { f, err := os.Create(*outputFile) errx(err) - data.Package = pkg + "_test" - if *prefix != "" { - data.PackagePrefix = *prefix - } else { - data.PackagePrefix = packageTestFunctionPrefix(pkg) + "Backend" - } + data.Package = pkg data.Funcs = findTestFunctions() generateOutput(f, data) diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index b2ee5e7da..4519ee25c 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -1,3 +1,4 @@ +// Package test contains a test suite for restic backends. package test import ( @@ -19,112 +20,88 @@ import ( "restic/backend" ) -// CreateFn is a function that creates a temporary repository for the tests. -var CreateFn func() (restic.Backend, error) +// Suite implements a test suite for restic backends. +type Suite struct { + Config interface{} -// OpenFn is a function that opens a previously created temporary repository. -var OpenFn func() (restic.Backend, error) + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig func() (interface{}, error) -// CleanupFn removes temporary files and directories created during the tests. -var CleanupFn func() error + // CreateFn is a function that creates a temporary repository for the tests. + Create func(cfg interface{}) (restic.Backend, error) -var but restic.Backend // backendUnderTest -var butInitialized bool + // OpenFn is a function that opens a previously created temporary repository. + Open func(cfg interface{}) (restic.Backend, error) -func open(t testing.TB) restic.Backend { - if OpenFn == nil { - t.Fatal("OpenFn not set") - } + // CleanupFn removes data created during the tests. + Cleanup func(cfg interface{}) error - if CreateFn == nil { - t.Fatalf("CreateFn not set") - } - - if !butInitialized { - be, err := CreateFn() - if err != nil { - t.Fatalf("Create returned unexpected error: %+v", err) - } - - but = be - butInitialized = true - } - - if but == nil { - var err error - but, err = OpenFn() - if err != nil { - t.Fatalf("Open returned unexpected error: %+v", err) - } - } - - return but + // MinimalData instructs the tests to not use excessive data. + MinimalData bool } -func close(t testing.TB) { - if but == nil { - t.Fatalf("trying to close non-existing backend") - } - - err := but.Close() +// RunTests executes all defined tests as subtests of t. +func (s *Suite) RunTests(t *testing.T) { + var err error + s.Config, err = s.NewConfig() if err != nil { - t.Fatalf("Close returned unexpected error: %+v", err) + t.Fatal(err) } - but = nil -} + // test create/open functions first + be := s.create(t) + s.close(t, be) -// TestCreate creates a backend. -func TestCreate(t testing.TB) { - if CreateFn == nil { - t.Fatalf("CreateFn not set!") + for _, test := range testFunctions { + t.Run(test.Name, func(t *testing.T) { + test.Fn(t, s) + }) } - be, err := CreateFn() - if err != nil { - t.Fatalf("Create returned error: %+v", err) + if !test.TestCleanupTempDirs { + t.Logf("not cleaning up backend") + return } - butInitialized = true - - err = be.Close() - if err != nil { - t.Fatalf("Close returned error: %+v", err) + if err = s.Cleanup(s.Config); err != nil { + t.Fatal(err) } } -// TestOpen opens a previously created backend. -func TestOpen(t testing.TB) { - if OpenFn == nil { - t.Fatalf("OpenFn not set!") - } - - be, err := OpenFn() +func (s *Suite) create(t testing.TB) restic.Backend { + be, err := s.Create(s.Config) if err != nil { - t.Fatalf("Open returned error: %+v", err) + t.Fatal(err) } + return be +} - err = be.Close() +func (s *Suite) open(t testing.TB) restic.Backend { + be, err := s.Open(s.Config) if err != nil { - t.Fatalf("Close returned error: %+v", err) + t.Fatal(err) + } + return be +} + +func (s *Suite) close(t testing.TB, be restic.Backend) { + err := be.Close() + if err != nil { + t.Fatal(err) } } -// TestCreateWithConfig tests that creating a backend in a location which already +// BackendTestCreateWithConfig tests that creating a backend in a location which already // has a config file fails. -func TestCreateWithConfig(t testing.TB) { - if CreateFn == nil { - t.Fatalf("CreateFn not set") - } - - b := open(t) - defer close(t) +func BackendTestCreateWithConfig(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) // save a config store(t, b, restic.ConfigFile, []byte("test config")) // now create the backend again, this must fail - _, err := CreateFn() + _, err := s.Create(s.Config) if err == nil { t.Fatalf("expected error not found for creating a backend with an existing config file") } @@ -136,10 +113,10 @@ func TestCreateWithConfig(t testing.TB) { } } -// TestLocation tests that a location string is returned. -func TestLocation(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestLocation tests that a location string is returned. +func BackendTestLocation(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) l := b.Location() if l == "" { @@ -147,10 +124,10 @@ func TestLocation(t testing.TB) { } } -// TestConfig saves and loads a config from the backend. -func TestConfig(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestConfig saves and loads a config from the backend. +func BackendTestConfig(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) var testString = "Config" @@ -180,10 +157,10 @@ func TestConfig(t testing.TB) { } } -// TestLoad tests the backend's Load function. -func TestLoad(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestLoad tests the backend's Load function. +func BackendTestLoad(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) _, err := b.Load(restic.Handle{}, 0, 0) if err == nil { @@ -215,7 +192,12 @@ func TestLoad(t testing.TB) { t.Fatalf("Load() returned a non-nil reader for negative offset!") } - for i := 0; i < 50; i++ { + loadTests := 50 + if s.MinimalData { + loadTests = 10 + } + + for i := 0; i < loadTests; i++ { l := rand.Intn(length + 2000) o := rand.Intn(length + 2000) @@ -245,31 +227,41 @@ func TestLoad(t testing.TB) { buf, err := ioutil.ReadAll(rd) if err != nil { t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err) - rd.Close() + if err = rd.Close(); err != nil { + t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err) + } continue } if l == 0 && len(buf) != len(d) { t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, len(d), len(buf)) - rd.Close() + if err = rd.Close(); err != nil { + t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err) + } continue } if l > 0 && l <= len(d) && len(buf) != l { t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf)) - rd.Close() + if err = rd.Close(); err != nil { + t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err) + } continue } if l > len(d) && len(buf) != len(d) { t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf)) - rd.Close() + if err = rd.Close(); err != nil { + t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err) + } continue } if !bytes.Equal(buf, d) { t.Errorf("Load(%d, %d) returned wrong bytes", l, o) - rd.Close() + if err = rd.Close(); err != nil { + t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err) + } continue } @@ -293,13 +285,18 @@ func (ec errorCloser) Close() error { return errors.New("forbidden method close was called") } -// TestSave tests saving data in the backend. -func TestSave(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestSave tests saving data in the backend. +func BackendTestSave(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) var id restic.ID - for i := 0; i < 10; i++ { + saveTests := 10 + if s.MinimalData { + saveTests = 2 + } + + for i := 0; i < saveTests; i++ { length := rand.Intn(1<<23) + 200000 data := test.Random(23, length) // use the first 32 byte as the ID @@ -388,10 +385,10 @@ var filenameTests = []struct { }, } -// TestSaveFilenames tests saving data with various file names in the backend. -func TestSaveFilenames(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestSaveFilenames tests saving data with various file names in the backend. +func BackendTestSaveFilenames(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) for i, test := range filenameTests { h := restic.Handle{Name: test.name, Type: restic.DataFile} @@ -437,10 +434,10 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res return h } -// TestBackend tests all functions of the backend. -func TestBackend(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestBackend tests all functions of the backend. +func BackendTestBackend(t testing.TB, s *Suite) { + b := s.open(t) + defer s.close(t, b) for _, tpe := range []restic.FileType{ restic.DataFile, restic.KeyFile, restic.LockFile, @@ -571,10 +568,14 @@ func TestBackend(t testing.TB) { } } -// TestDelete tests the Delete function. -func TestDelete(t testing.TB) { - b := open(t) - defer close(t) +// BackendTestDelete tests the Delete function. +func BackendTestDelete(t testing.TB, s *Suite) { + if !test.TestCleanupTempDirs { + t.Skipf("not removing backend, TestCleanupTempDirs is false") + } + + b := s.open(t) + defer s.close(t, b) be, ok := b.(restic.Deleter) if !ok { @@ -586,21 +587,3 @@ func TestDelete(t testing.TB) { t.Fatalf("error deleting backend: %+v", err) } } - -// TestCleanup runs the cleanup function after all tests are run. -func TestCleanup(t testing.TB) { - if CleanupFn == nil { - t.Log("CleanupFn function not set") - return - } - - if !test.TestCleanupTempDirs { - t.Logf("not cleaning up backend") - return - } - - err := CleanupFn() - if err != nil { - t.Fatalf("Cleanup returned error: %+v", err) - } -} diff --git a/src/restic/backend/test/tests_test.go b/src/restic/backend/test/tests_test.go index 04e9936e0..514cd9ef9 100644 --- a/src/restic/backend/test/tests_test.go +++ b/src/restic/backend/test/tests_test.go @@ -2,38 +2,59 @@ package test_test import ( "restic" - "restic/errors" + "testing" "restic/backend/mem" "restic/backend/test" ) -var be restic.Backend +//go:generate go run generate_test_list.go -//go:generate go run ../test/generate_backend_tests.go - -func init() { - test.CreateFn = func() (restic.Backend, error) { - if be != nil { - return nil, errors.New("temporary memory backend dir already exists") - } - - be = mem.New() - - return be, nil - } - - test.OpenFn = func() (restic.Backend, error) { - if be == nil { - return nil, errors.New("repository not initialized") - } - - return be, nil - } - - test.CleanupFn = func() error { - be = nil - return nil - } +type memConfig struct { + be restic.Backend +} + +func TestSuiteBackendMem(t *testing.T) { + suite := test.Suite{ + // NewConfig returns a config for a new temporary backend that will be used in tests. + NewConfig: func() (interface{}, error) { + return &memConfig{}, nil + }, + + // CreateFn is a function that creates a temporary repository for the tests. + Create: func(cfg interface{}) (restic.Backend, error) { + c := cfg.(*memConfig) + if c.be != nil { + ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile}) + if err != nil { + return nil, err + } + + if ok { + return nil, errors.New("config already exists") + } + } + + c.be = mem.New() + return c.be, nil + }, + + // OpenFn is a function that opens a previously created temporary repository. + Open: func(cfg interface{}) (restic.Backend, error) { + c := cfg.(*memConfig) + if c.be == nil { + c.be = mem.New() + } + return c.be, nil + }, + + // CleanupFn removes data created during the tests. + Cleanup: func(cfg interface{}) error { + // no cleanup needed + return nil + }, + } + + suite.RunTests(t) } diff --git a/src/restic/backend/utils.go b/src/restic/backend/utils.go index 27d2b9ad5..3f3a85749 100644 --- a/src/restic/backend/utils.go +++ b/src/restic/backend/utils.go @@ -14,8 +14,12 @@ func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) { } defer func() { - io.Copy(ioutil.Discard, rd) - e := rd.Close() + _, e := io.Copy(ioutil.Discard, rd) + if err == nil { + err = e + } + + e = rd.Close() if err == nil { err = e } diff --git a/src/restic/debug/round_tripper_debug.go b/src/restic/debug/round_tripper_debug.go new file mode 100644 index 000000000..0ff9343ad --- /dev/null +++ b/src/restic/debug/round_tripper_debug.go @@ -0,0 +1,92 @@ +// +build debug + +package debug + +import ( + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httputil" + "os" + "restic/errors" +) + +type eofDetectRoundTripper struct { + http.RoundTripper +} + +type eofDetectReader struct { + eofSeen bool + rd io.ReadCloser +} + +func (rd *eofDetectReader) Read(p []byte) (n int, err error) { + n, err = rd.rd.Read(p) + if err == io.EOF { + rd.eofSeen = true + } + return n, err +} + +func (rd *eofDetectReader) Close() error { + if !rd.eofSeen { + buf, err := ioutil.ReadAll(rd) + msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf)) + if err != nil { + msg += fmt.Sprintf(", error: %v", err) + } + + if len(buf) > 0 { + if len(buf) > 20 { + buf = append(buf[:20], []byte("...")...) + } + msg += fmt.Sprintf(", body: %q", buf) + } + + fmt.Fprintln(os.Stderr, msg) + Log("%s: %+v", msg, errors.New("Close()")) + } + return rd.rd.Close() +} + +func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + res, err = tr.RoundTripper.RoundTrip(req) + res.Body = &eofDetectReader{rd: res.Body} + return res, err +} + +type loggingRoundTripper struct { + http.RoundTripper +} + +// RoundTripper returns a new http.RoundTripper which logs all requests (if +// debug is enabled). When debug is not enabled, upstream is returned. +func RoundTripper(upstream http.RoundTripper) http.RoundTripper { + return loggingRoundTripper{eofDetectRoundTripper{upstream}} +} + +func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) { + trace, err := httputil.DumpRequestOut(req, false) + if err != nil { + Log("DumpRequestOut() error: %v\n", err) + } else { + Log("------------ HTTP REQUEST -----------\n%s", trace) + } + + res, err = tr.RoundTripper.RoundTrip(req) + if err != nil { + Log("RoundTrip() returned error: %v", err) + } + + if res != nil { + trace, err := httputil.DumpResponse(res, false) + if err != nil { + Log("DumpResponse() error: %v\n", err) + } else { + Log("------------ HTTP RESPONSE ----------\n%s", trace) + } + } + + return res, err +} diff --git a/src/restic/debug/round_tripper_release.go b/src/restic/debug/round_tripper_release.go new file mode 100644 index 000000000..6efff2c28 --- /dev/null +++ b/src/restic/debug/round_tripper_release.go @@ -0,0 +1,11 @@ +// +build !debug + +package debug + +import "net/http" + +// RoundTripper returns a new http.RoundTripper which logs all requests (if +// debug is enabled). When debug is not enabled, upstream is returned. +func RoundTripper(upstream http.RoundTripper) http.RoundTripper { + return upstream +} diff --git a/src/restic/test/helpers.go b/src/restic/test/helpers.go index a6dffc0a4..d208d52dc 100644 --- a/src/restic/test/helpers.go +++ b/src/restic/test/helpers.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "reflect" + "restic/errors" "runtime" "testing" @@ -122,7 +123,7 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) { // Env creates a test environment and extracts the repository fixture. // Returned is the repo path and a cleanup function. func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) { - tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") + tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-env-") OK(t, err) fd, err := os.Open(repoFixture) @@ -151,7 +152,11 @@ func isFile(fi os.FileInfo) bool { // This is mainly used for tests on Windows, which is unable to delete a file // set read-only. func ResetReadOnly(t testing.TB, dir string) { - OK(t, filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error { + if fi == nil { + return err + } + if fi.IsDir() { return os.Chmod(path, 0777) } @@ -161,14 +166,22 @@ func ResetReadOnly(t testing.TB, dir string) { } return nil - })) + }) + if os.IsNotExist(errors.Cause(err)) { + err = nil + } + OK(t, err) } // RemoveAll recursively resets the read-only flag of all files and dirs and // afterwards uses os.RemoveAll() to remove the path. func RemoveAll(t testing.TB, path string) { ResetReadOnly(t, path) - OK(t, os.RemoveAll(path)) + err := os.RemoveAll(path) + if os.IsNotExist(errors.Cause(err)) { + err = nil + } + OK(t, err) } // TempDir returns a temporary directory that is removed when cleanup is