mirror
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #938 from restic/rework-backend-tests 

WIP: rework backend integration tests
This commit is contained in:
Alexander Neumann 2017-05-12 22:09:57 +02:00
parent c5ae5524ff ec5e984ed9
commit 19daefd04e
28 changed files with 890 additions and 1176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
doc/tutorial_aws_s3.rst
View File

@ -1,12 +1,12 @@
Setting up Restic with Amazon S3
Setting up restic with Amazon S3
================================
Preface
-------
This tutorial will show you how to use Restic with AWS S3. It will show you how
This tutorial will show you how to use restic with AWS S3. It will show you how
to navigate the AWS web interface, create an S3 bucket, create a user with
access to only this bucket, and finally how to connect Restic to this bucket.
access to only this bucket, and finally how to connect restic to this bucket.
Prerequisites
-------------
@ -95,8 +95,8 @@ AWS through the ``restic`` program and not through the web interface. Therefore,
:alt: Choose User Name and Access Type
During the next step, permissions can be assigned to the new user. To use this
user with Restic, it only needs access to the ``restic-demo`` bucket. Select
"Attach exiting policies directly", which will bring up a list of pre-defined
user with restic, it only needs access to the ``restic-demo`` bucket. Select
"Attach existing policies directly", which will bring up a list of pre-defined
policies below. Afterwards, click the "Create policy" button to create a custom
policy:
@ -111,17 +111,17 @@ Generator" will be used to generate a policy file using a web interface:
:alt: Create a New Policy
After invoking the policy generator, you will be presented with a user
interface to generate individual permission statements. For Restic to work, two
interface to generate individual permission statements. For restic to work, two
such statements must be created. The first statement is set up as follows:
.. code::
Effect: Allow
Service: S3
Service: Amazon S3
Actions: DeleteObject, GetObject, PutObject
Resource: arn:aws:s3:::restic-demo/*
This statement allows Restic to create, read and delete objects inside the S3
This statement allows restic to create, read and delete objects inside the S3
bucket named ``restic-demo``. Adjust the bucket's name to the name of the bucket
you created earlier. Using the "Add Statement" button, this statement can be
saved. Now a second statement is created:
@ -129,13 +129,13 @@ saved. Now a second statement is created:
.. code::
Effect: Allow
Service: S3
Service: Amazon S3
Actions: ListBucket
Resource: arn:aws:s3:::restic-demo
Again, substitute ``restic-demo`` with the actual name of your bucket. Note that,
unlike before, there is no ``/*`` after the bucket name. This statement allows
Restic to list the objects stored in the ``restic-demo`` bucket. Again, use "Add
restic to list the objects stored in the ``restic-demo`` bucket. Again, use "Add
Statement" to save this statement. The policy creator interface should now
look as follows:
@ -176,7 +176,7 @@ You have now completed the configuration in AWS. Feel free to close your web
browser now.
Initializing the Restic repository
Initializing the restic repository
----------------------------------
Open a terminal and make sure you have the ``restic`` binary ready. First, choose
@ -189,7 +189,7 @@ this purpose:
I9n7G7G0ZpDWA3GOcJbIuwQCGvGUBkU5
Note this password somewhere safe along with your AWS credentials. Next, the
configuration of Restic will be placed into environment variables. This will
configuration of restic will be placed into environment variables. This will
include sensitive information, such as your AWS secret and repository password.
Therefore, make sure the next commands **do not** end up in your shell's
history file. Adjust the contents of the environment variables to fit your
@ -204,7 +204,7 @@ bucket's name and your user's API credentials.
$ export RESTIC_PASSWORD="I9n7G7G0ZpDWA3GOcJbIuwQCGvGUBkU5"
After the environment is set up, Restic may be called to initialize the
After the environment is set up, restic may be called to initialize the
repository:
@ -217,7 +217,7 @@ repository:
the repository. Losing your password means that your data is
irrecoverably lost.
Restic is now ready to be used with AWS S3. Try to create a backup:
restic is now ready to be used with AWS S3. Try to create a backup:
.. code-block:: console

229
run_integration_tests.go
View File

@ -9,7 +9,6 @@ import (
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"os/exec"
@ -25,21 +24,6 @@ var ForbiddenImports = map[string]bool{
}
var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests")
var minioServer = flag.String("minio", "", "path to the minio server binary")
var restServer = flag.String("rest", "", "path to the rest-server binary")
var debug = flag.Bool("debug", false, "output debug messages")
var minioServerEnv = map[string]string{
"MINIO_ACCESS_KEY": "KEBIYDZ87HCIH5D17YCN",
"MINIO_SECRET_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
var minioEnv = map[string]string{
"RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000",
"RESTIC_TEST_REST_SERVER": "http://127.0.0.1:8000",
"AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN",
"AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe",
}
func init() {
flag.Parse()
@ -55,32 +39,11 @@ type CIEnvironment interface {
// TravisEnvironment is the environment in which Travis tests run.
type TravisEnvironment struct {
goxOSArch []string
minio string
minioSrv *Background
minioTempdir string
rest string
restSrv *Background
restTempdir string
env map[string]string
env map[string]string
}
func (env *TravisEnvironment) getMinio() error {
if *minioServer != "" {
msg("using minio server at %q\n", *minioServer)
env.minio = *minioServer
return nil
}
if *restServer != "" {
msg("using REST server at %q\n", *restServer)
env.rest = *restServer
return nil
}
tempfile, err := ioutil.TempFile("", "minio-server-")
tempfile, err := os.Create(filepath.Join(os.Getenv("GOPATH"), "bin", "minio"))
if err != nil {
return fmt.Errorf("create tempfile for minio download failed: %v\n", err)
}
@ -115,64 +78,6 @@ func (env *TravisEnvironment) getMinio() error {
}
msg("downloaded minio server to %v\n", tempfile.Name())
env.minio = tempfile.Name()
return nil
}
func (env *TravisEnvironment) runMinio() error {
if env.minio == "" {
return nil
}
// start minio server
msg("starting minio server at %s", env.minio)
dir, err := ioutil.TempDir("", "minio-root")
if err != nil {
return fmt.Errorf("TempDir: %v", err)
}
env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio,
"server",
"--address", "127.0.0.1:9000",
dir)
if err != nil {
return fmt.Errorf("error running minio server: %v", err)
}
// go func() {
// time.Sleep(300 * time.Millisecond)
// env.minioSrv.Cmd.Process.Kill()
// }()
for k, v := range minioEnv {
env.env[k] = v
}
env.minioTempdir = dir
return nil
}
func (env *TravisEnvironment) runRESTServer() error {
if env.rest == "" {
return nil
}
// start rest server
msg("starting rest server at %s", env.rest)
dir, err := ioutil.TempDir("", "rest-server-root")
if err != nil {
return fmt.Errorf("TempDir: %v", err)
}
env.restSrv, err = StartBackgroundCommand(map[string]string{}, env.rest,
"--path", dir)
if err != nil {
return fmt.Errorf("error running rest server: %v", err)
}
env.restTempdir = dir
return nil
}
@ -186,10 +91,7 @@ func (env *TravisEnvironment) Prepare() error {
"golang.org/x/tools/cmd/cover",
"github.com/pierrre/gotestcover",
"github.com/NebulousLabs/glyphcheck",
}
if env.rest == "" {
pkgs = append(pkgs, "github.com/restic/rest-server")
"github.com/restic/rest-server",
}
for _, pkg := range pkgs {
@ -199,19 +101,9 @@ func (env *TravisEnvironment) Prepare() error {
}
}
if env.rest == "" {
env.rest = filepath.Join(os.Getenv("GOPATH"), "bin", "rest-server")
}
if err := env.getMinio(); err != nil {
return err
}
if err := env.runMinio(); err != nil {
return err
}
if err := env.runRESTServer(); err != nil {
return err
}
if *runCrossCompile {
// only test cross compilation on linux with Travis
@ -240,110 +132,9 @@ func (env *TravisEnvironment) Prepare() error {
// Teardown stops backend services and cleans the environment again.
func (env *TravisEnvironment) Teardown() error {
msg("run travis teardown\n")
if env.minioSrv != nil {
msg("stopping minio server\n")
if env.minioSrv.Cmd.ProcessState == nil {
err := env.minioSrv.Cmd.Process.Kill()
if err != nil {
fmt.Fprintf(os.Stderr, "error killing minio server process: %v", err)
}
} else {
result := <-env.minioSrv.Result
if result.Error != nil {
msg("minio server returned error: %v\n", result.Error)
msg("stdout: %s\n", result.Stdout)
msg("stderr: %s\n", result.Stderr)
}
}
err := os.RemoveAll(env.minioTempdir)
if err != nil {
msg("error removing minio tempdir %v: %v\n", env.minioTempdir, err)
}
}
if env.restSrv != nil {
msg("stopping rest-server\n")
if env.restSrv.Cmd.ProcessState == nil {
err := env.restSrv.Cmd.Process.Kill()
if err != nil {
fmt.Fprintf(os.Stderr, "error killing rest-server process: %v", err)
}
} else {
result := <-env.restSrv.Result
if result.Error != nil {
msg("rest-server returned error: %v\n", result.Error)
msg("stdout: %s\n", result.Stdout)
msg("stderr: %s\n", result.Stderr)
}
}
err := os.RemoveAll(env.restTempdir)
if err != nil {
msg("error removing rest-server tempdir %v: %v\n", env.restTempdir, err)
}
}
return nil
}
// Background is a program running in the background.
type Background struct {
Cmd *exec.Cmd
Result chan Result
}
// Result is the result of a program that ran in the background.
type Result struct {
Stdout, Stderr string
Error error
}
// StartBackgroundCommand runs a program in the background.
func StartBackgroundCommand(env map[string]string, cmd string, args ...string) (*Background, error) {
msg("running background command %v %v\n", cmd, args)
b := Background{
Result: make(chan Result, 1),
}
stdout := bytes.NewBuffer(nil)
stderr := bytes.NewBuffer(nil)
c := exec.Command(cmd, args...)
c.Stdout = stdout
c.Stderr = stderr
if *debug {
c.Stdout = io.MultiWriter(c.Stdout, os.Stdout)
c.Stderr = io.MultiWriter(c.Stderr, os.Stderr)
}
c.Env = updateEnv(os.Environ(), env)
b.Cmd = c
err := c.Start()
if err != nil {
msg("error starting background job %v: %v\n", cmd, err)
return nil, err
}
go func() {
err := b.Cmd.Wait()
msg("background job %v returned: %v\n", cmd, err)
msg("stdout: %s\n", stdout.Bytes())
msg("stderr: %s\n", stderr.Bytes())
b.Result <- Result{
Stdout: string(stdout.Bytes()),
Stderr: string(stderr.Bytes()),
Error: err,
}
}()
return &b, nil
}
// RunTests starts the tests for Travis.
func (env *TravisEnvironment) RunTests() error {
// do not run fuse tests on darwin
@ -436,16 +227,16 @@ func (env *AppveyorEnvironment) Teardown() error {
// findGoFiles returns a list of go source code file names below dir.
func findGoFiles(dir string) (list []string, err error) {
err = filepath.Walk(dir, func(name string, fi os.FileInfo, err error) error {
if filepath.Base(name) == "vendor" {
relpath, err := filepath.Rel(dir, name)
if err != nil {
return err
}
if relpath == "vendor" || relpath == "pkg" {
return filepath.SkipDir
}
if filepath.Ext(name) == ".go" {
relpath, err := filepath.Rel(dir, name)
if err != nil {
return err
}
if filepath.Ext(relpath) == ".go" {
list = append(list, relpath)
}

19
src/cmds/restic/main.go
View File

@ -1,7 +1,10 @@
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"os"
"restic"
"restic/debug"
@ -44,6 +47,14 @@ directories in an encrypted repository stored on different backends.
},
}
var logBuffer = bytes.NewBuffer(nil)
func init() {
// install custom global logger into a buffer, if an error occurs
// we can show the logs
log.SetOutput(logBuffer)
}
func main() {
debug.Log("main %#v", os.Args)
err := cmdRoot.Execute()
@ -55,6 +66,14 @@ func main() {
fmt.Fprintf(os.Stderr, "%v\n", err)
case err != nil:
fmt.Fprintf(os.Stderr, "%+v\n", err)
if logBuffer.Len() > 0 {
fmt.Fprintf(os.Stderr, "also, the following messages were logged by a library:\n")
sc := bufio.NewScanner(logBuffer)
for sc.Scan() {
fmt.Fprintln(os.Stderr, sc.Text())
}
}
}
var exitCode int

28
src/restic/backend/http_transport.go Normal file
View File

@ -0,0 +1,28 @@
package backend
import (
"net"
"net/http"
"restic/debug"
"time"
)
// Transport returns a new http.RoundTripper with default settings applied.
func Transport() http.RoundTripper {
// copied from net/http
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
// wrap in the debug round tripper
return debug.RoundTripper(tr)
}

87
src/restic/backend/local/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package local_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestLocalBackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestLocalBackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestLocalBackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestLocalBackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestLocalBackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestLocalBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestLocalBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestLocalBackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestLocalBackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestLocalBackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestLocalBackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

76
src/restic/backend/local/local_test.go
View File

@ -1,59 +1,55 @@
package local_test
import (
"fmt"
"io/ioutil"
"os"
"restic"
"testing"
"restic/backend/local"
"restic/backend/test"
. "restic/test"
)
var tempBackendDir string
func TestBackend(t *testing.T) {
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
dir, err := ioutil.TempDir(TestTempDir, "restic-test-local-")
if err != nil {
t.Fatal(err)
}
//go:generate go run ../test/generate_backend_tests.go
t.Logf("create new backend at %v", dir)
func createTempdir() error {
if tempBackendDir != "" {
return nil
}
cfg := local.Config{
Path: dir,
}
return cfg, nil
},
tempdir, err := ioutil.TempDir("", "restic-local-test-")
if err != nil {
return err
}
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Create(cfg)
},
fmt.Printf("created new test backend at %v\n", tempdir)
tempBackendDir = tempdir
return nil
}
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(local.Config)
return local.Open(cfg)
},
func init() {
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
}
return local.Create(local.Config{Path: tempBackendDir})
}
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(local.Config)
if !TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}
test.OpenFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
}
return local.Open(local.Config{Path: tempBackendDir})
}
test.CleanupFn = func() error {
if tempBackendDir == "" {
RemoveAll(t, cfg.Path)
return nil
}
fmt.Printf("removing test backend at %v\n", tempBackendDir)
err := os.RemoveAll(tempBackendDir)
tempBackendDir = ""
return err
},
}
suite.RunTests(t)
}

87
src/restic/backend/mem/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package mem_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestMemBackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestMemBackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestMemBackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestMemBackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestMemBackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestMemBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestMemBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestMemBackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestMemBackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestMemBackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestMemBackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

74
src/restic/backend/mem/mem_backend_test.go
View File

@ -2,6 +2,7 @@ package mem_test
import (
"restic"
"testing"
"restic/errors"
@ -9,31 +10,50 @@ import (
"restic/backend/test"
)
var be restic.Backend
//go:generate go run ../test/generate_backend_tests.go
func init() {
test.CreateFn = func() (restic.Backend, error) {
if be != nil {
return nil, errors.New("temporary memory backend dir already exists")
}
be = mem.New()
return be, nil
}
test.OpenFn = func() (restic.Backend, error) {
if be == nil {
return nil, errors.New("repository not initialized")
}
return be, nil
}
test.CleanupFn = func() error {
be = nil
return nil
}
type memConfig struct {
be restic.Backend
}
func TestSuiteBackendMem(t *testing.T) {
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
return &memConfig{}, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be != nil {
ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if ok {
return nil, errors.New("config already exists")
}
}
c.be = mem.New()
return c.be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be == nil {
c.be = mem.New()
}
return c.be, nil
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg interface{}) error {
// no cleanup needed
return nil
},
}
suite.RunTests(t)
}

87
src/restic/backend/rest/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package rest_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestRestBackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestRestBackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestRestBackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestRestBackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestRestBackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestRestBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestRestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestRestBackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestRestBackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestRestBackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestRestBackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

4
src/restic/backend/rest/rest.go
View File

@ -35,8 +35,8 @@ func Open(cfg Config) (restic.Backend, error) {
for i := 0; i < connLimit; i++ {
connChan <- struct{}{}
}
tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
client := http.Client{Transport: tr}
client := http.Client{Transport: backend.Transport()}
// use url without trailing slash for layout
url := cfg.URL.String()

110
src/restic/backend/rest/rest_test.go
View File

@ -1,39 +1,113 @@
package rest_test
import (
"fmt"
"context"
"io/ioutil"
"net"
"net/url"
"os"
"os/exec"
"restic"
"testing"
"time"
"restic/backend/rest"
"restic/backend/test"
. "restic/test"
)
//go:generate go run ../test/generate_backend_tests.go
func init() {
if TestRESTServer == "" {
SkipMessage = "REST test server not available"
return
}
url, err := url.Parse(TestRESTServer)
func runRESTServer(ctx context.Context, t testing.TB, dir string) func() {
srv, err := exec.LookPath("rest-server")
if err != nil {
fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
return
t.Skip(err)
}
cfg := rest.Config{
URL: url,
cmd := exec.CommandContext(ctx, srv, "--path", dir)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stdout
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
test.CreateFn = func() (restic.Backend, error) {
return rest.Create(cfg)
// wait until the TCP port is reachable
var success bool
for i := 0; i < 10; i++ {
time.Sleep(200 * time.Millisecond)
c, err := net.Dial("tcp", "localhost:8000")
if err != nil {
continue
}
success = true
if err := c.Close(); err != nil {
t.Fatal(err)
}
}
test.OpenFn = func() (restic.Backend, error) {
return rest.Open(cfg)
if !success {
t.Fatal("unable to connect to rest server")
return nil
}
return func() {
if err := cmd.Process.Kill(); err != nil {
t.Fatal(err)
}
// ignore errors, we've killed the process
_ = cmd.Wait()
}
}
func TestBackend(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
dir, cleanup := TempDir(t)
defer cleanup()
cleanup = runRESTServer(ctx, t, dir)
defer cleanup()
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
dir, err := ioutil.TempDir(TestTempDir, "restic-test-rest-")
if err != nil {
t.Fatal(err)
}
t.Logf("create new backend at %v", dir)
url, err := url.Parse("http://localhost:8000/restic-test")
if err != nil {
t.Fatal(err)
}
cfg := rest.Config{
URL: url,
}
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Create(cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(rest.Config)
return rest.Open(cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
return nil
},
}
suite.RunTests(t)
}

87
src/restic/backend/s3/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package s3_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestS3BackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestS3BackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestS3BackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestS3BackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestS3BackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestS3BackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestS3BackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestS3BackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestS3BackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestS3BackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestS3BackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

4
src/restic/backend/s3/s3.go
View File

@ -3,7 +3,6 @@ package s3
import (
"bytes"
"io"
"net/http"
"path"
"restic"
"strings"
@ -48,8 +47,7 @@ func Open(cfg Config) (restic.Backend, error) {
Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join},
}
tr := &http.Transport{MaxIdleConnsPerHost: connLimit}
client.SetCustomTransport(tr)
client.SetCustomTransport(backend.Transport())
be.createConnections()

291
src/restic/backend/s3/s3_test.go
View File

@ -1,73 +1,260 @@
package s3_test
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"net/url"
"io"
"net"
"os"
"os/exec"
"path/filepath"
"restic"
"restic/errors"
"testing"
"time"
"restic/backend/s3"
"restic/backend/test"
. "restic/test"
)
//go:generate go run ../test/generate_backend_tests.go
func init() {
if TestS3Server == "" {
SkipMessage = "s3 test server not available"
return
}
url, err := url.Parse(TestS3Server)
func mkdir(t testing.TB, dir string) {
err := os.MkdirAll(dir, 0700)
if err != nil {
fmt.Fprintf(os.Stderr, "invalid url: %v\n", err)
t.Fatal(err)
}
}
func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func() {
mkdir(t, filepath.Join(dir, "config"))
mkdir(t, filepath.Join(dir, "root"))
cmd := exec.CommandContext(ctx, "minio",
"server",
"--address", "127.0.0.1:9000",
"--config-dir", filepath.Join(dir, "config"),
filepath.Join(dir, "root"))
cmd.Env = append(os.Environ(),
"MINIO_ACCESS_KEY="+key,
"MINIO_SECRET_KEY="+secret,
)
cmd.Stderr = os.Stderr
err := cmd.Start()
if err != nil {
t.Fatal(err)
}
// wait until the TCP port is reachable
var success bool
for i := 0; i < 10; i++ {
time.Sleep(200 * time.Millisecond)
c, err := net.Dial("tcp", "localhost:9000")
if err != nil {
continue
}
success = true
if err := c.Close(); err != nil {
t.Fatal(err)
}
}
if !success {
t.Fatal("unable to connect to minio server")
return nil
}
return func() {
err = cmd.Process.Kill()
if err != nil {
t.Fatal(err)
}
// ignore errors, we've killed the process
_ = cmd.Wait()
}
}
func newCredentials(t testing.TB) (key, secret string) {
buf := make([]byte, 10)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
t.Fatal(err)
}
key = hex.EncodeToString(buf)
_, err = io.ReadFull(rand.Reader, buf)
if err != nil {
t.Fatal(err)
}
secret = hex.EncodeToString(buf)
return key, secret
}
func TestBackendMinio(t *testing.T) {
// try to find a minio binary
_, err := exec.LookPath("minio")
if err != nil {
t.Skip(err)
return
}
cfg := s3.Config{
Endpoint: url.Host,
Bucket: "restictestbucket",
KeyID: os.Getenv("AWS_ACCESS_KEY_ID"),
Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"),
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type Config struct {
s3.Config
tempdir string
removeTempdir func()
stopServer func()
}
if url.Scheme == "http" {
cfg.UseHTTP = true
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
cfg := Config{}
cfg.tempdir, cfg.removeTempdir = TempDir(t)
key, secret := newCredentials(t)
cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
cfg.Config = s3.Config{
Endpoint: "localhost:9000",
Bucket: "restictestbucket",
Prefix: fmt.Sprintf("test-%d", time.Now().UnixNano()),
UseHTTP: true,
KeyID: key,
Secret: secret,
}
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(Config)
be, err := s3.Open(cfg.Config)
if err != nil {
return nil, err
}
exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(Config)
return s3.Open(cfg.Config)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(Config)
if cfg.stopServer != nil {
cfg.stopServer()
}
if cfg.removeTempdir != nil {
t.Logf("removeTempdir %v", config)
cfg.removeTempdir()
}
return nil
},
}
test.CreateFn = func() (restic.Backend, error) {
be, err := s3.Open(cfg)
if err != nil {
return nil, err
}
exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
}
test.OpenFn = func() (restic.Backend, error) {
return s3.Open(cfg)
}
// test.CleanupFn = func() error {
// if tempBackendDir == "" {
// return nil
// }
// fmt.Printf("removing test backend at %v\n", tempBackendDir)
// err := os.RemoveAll(tempBackendDir)
// tempBackendDir = ""
// return err
// }
suite.RunTests(t)
}
func TestBackendS3(t *testing.T) {
vars := []string{
"RESTIC_TEST_S3_KEY",
"RESTIC_TEST_S3_SECRET",
"RESTIC_TEST_S3_REPOSITORY",
}
for _, v := range vars {
if os.Getenv(v) == "" {
t.Skipf("environment variable %v not set", v)
return
}
}
suite := test.Suite{
// do not use excessive data
MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
if err != nil {
return nil, err
}
cfg := s3cfg.(s3.Config)
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET")
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
be, err := s3.Open(cfg)
if err != nil {
return nil, err
}
exists, err := be.Test(restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
return s3.Open(cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(s3.Config)
be, err := s3.Open(cfg)
if err != nil {
return err
}
if err := be.(restic.Deleter).Delete(); err != nil {
return err
}
return nil
},
}
t.Logf("run tests")
suite.RunTests(t)
}

87
src/restic/backend/sftp/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package sftp_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestSftpBackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestSftpBackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestSftpBackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestSftpBackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestSftpBackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestSftpBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestSftpBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestSftpBackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestSftpBackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestSftpBackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestSftpBackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

4
src/restic/backend/sftp/layout_test.go
View File

@ -10,7 +10,7 @@ import (
)
func TestLayout(t *testing.T) {
if sftpserver == "" {
if sftpServer == "" {
t.Skip("sftp server binary not available")
}
@ -46,7 +46,7 @@ func TestLayout(t *testing.T) {
repo := filepath.Join(path, "repo")
be, err := sftp.Open(sftp.Config{
Command: fmt.Sprintf("%q -e", sftpserver),
Command: fmt.Sprintf("%q -e", sftpServer),
Path: repo,
Layout: test.layout,
})

2
src/restic/backend/sftp/sftp.go
View File

@ -294,7 +294,7 @@ func (r *SFTP) Save(h restic.Handle, rd io.Reader) (err error) {
// save data
_, err = io.Copy(f, rd)
if err != nil {
f.Close()
_ = f.Close()
return errors.Wrap(err, "Write")
}

92
src/restic/backend/sftp/sftp_backend_test.go
View File

@ -1,92 +0,0 @@
package sftp_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"restic"
"strings"
"restic/errors"
"restic/backend/sftp"
"restic/backend/test"
. "restic/test"
)
var tempBackendDir string
//go:generate go run ../test/generate_backend_tests.go
func createTempdir() error {
if tempBackendDir != "" {
return nil
}
tempdir, err := ioutil.TempDir("", "restic-local-test-")
if err != nil {
return err
}
tempBackendDir = tempdir
return nil
}
func findSFTPServerBinary() string {
for _, dir := range strings.Split(TestSFTPPath, ":") {
testpath := filepath.Join(dir, "sftp-server")
_, err := os.Stat(testpath)
if !os.IsNotExist(errors.Cause(err)) {
return testpath
}
}
return ""
}
var sftpserver = findSFTPServerBinary()
func init() {
if sftpserver == "" {
SkipMessage = "sftp server binary not found, skipping tests"
return
}
cfg := sftp.Config{
Command: fmt.Sprintf("%q -e", sftpserver),
}
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
}
cfg.Path = tempBackendDir
return sftp.Create(cfg)
}
test.OpenFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
}
cfg.Path = tempBackendDir
return sftp.Open(cfg)
}
test.CleanupFn = func() error {
if tempBackendDir == "" {
return nil
}
err := os.RemoveAll(tempBackendDir)
tempBackendDir = ""
return err
}
}

79
src/restic/backend/sftp/sftp_test.go Normal file
View File

@ -0,0 +1,79 @@
package sftp_test
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"restic"
"restic/backend/sftp"
"restic/backend/test"
"restic/errors"
"strings"
"testing"
. "restic/test"
)
func findSFTPServerBinary() string {
for _, dir := range strings.Split(TestSFTPPath, ":") {
testpath := filepath.Join(dir, "sftp-server")
_, err := os.Stat(testpath)
if !os.IsNotExist(errors.Cause(err)) {
return testpath
}
}
return ""
}
var sftpServer = findSFTPServerBinary()
func TestBackend(t *testing.T) {
if sftpServer == "" {
t.Skip("sftp server binary not found")
}
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
dir, err := ioutil.TempDir(TestTempDir, "restic-test-sftp-")
if err != nil {
t.Fatal(err)
}
t.Logf("create new backend at %v", dir)
cfg := sftp.Config{
Path: dir,
Command: fmt.Sprintf("%q -e", sftpServer),
}
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Create(cfg)
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(sftp.Config)
return sftp.Open(cfg)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(sftp.Config)
if !TestCleanupTempDirs {
t.Logf("leaving test backend dir at %v", cfg.Path)
}
RemoveAll(t, cfg.Path)
return nil
},
}
suite.RunTests(t)
}

87
src/restic/backend/test/backend_test.go
View File

@ -1,87 +0,0 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package test_test
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
func TestTestBackendCreate(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreate(t)
}
func TestTestBackendOpen(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestOpen(t)
}
func TestTestBackendCreateWithConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCreateWithConfig(t)
}
func TestTestBackendLocation(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLocation(t)
}
func TestTestBackendConfig(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestConfig(t)
}
func TestTestBackendLoad(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoad(t)
}
func TestTestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSave(t)
}
func TestTestBackendSaveFilenames(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestSaveFilenames(t)
}
func TestTestBackendBackend(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestBackend(t)
}
func TestTestBackendDelete(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestDelete(t)
}
func TestTestBackendCleanup(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestCleanup(t)
}

21
src/restic/backend/test/funcs.go Normal file
View File

@ -0,0 +1,21 @@
// DO NOT EDIT, AUTOMATICALLY GENERATED
package test
import (
"testing"
)
var testFunctions = []struct {
Name string
Fn func(t testing.TB, suite *Suite)
}{
{"CreateWithConfig", BackendTestCreateWithConfig},
{"Location", BackendTestLocation},
{"Config", BackendTestConfig},
{"Load", BackendTestLoad},
{"Save", BackendTestSave},
{"SaveFilenames", BackendTestSaveFilenames},
{"Backend", BackendTestBackend},
{"Delete", BackendTestDelete},
}

37
src/restic/backend/test/generate_backend_tests.go → src/restic/backend/test/generate_test_list.go
View File

@ -18,35 +18,31 @@ import (
)
var data struct {
Package string
PackagePrefix string
Funcs []string
Package string
Funcs []string
}
var testTemplate = `
// DO NOT EDIT, AUTOMATICALLY GENERATED
package {{ .Package }}
import (
"testing"
"restic/backend/test"
)
var SkipMessage string
{{ $prefix := .PackagePrefix }}
{{ range $f := .Funcs }}
func Test{{ $prefix }}{{ $f }}(t *testing.T){
if SkipMessage != "" { t.Skip(SkipMessage) }
test.Test{{ $f }}(t)
}
var testFunctions = []struct {
Name string
Fn func(t testing.TB, suite *Suite)
}{
{{ range $f := .Funcs -}}
{"{{ $f }}", BackendTest{{ $f }},},
{{ end }}
}
`
var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in")
var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to")
var testFile = flag.String("testfile", "tests.go", "file to search test functions in")
var outputFile = flag.String("output", "funcs.go", "output file to write generated code to")
var packageName = flag.String("package", "", "the package name to use")
var prefix = flag.String("prefix", "", "test function prefix")
var quiet = flag.Bool("quiet", false, "be quiet")
@ -60,7 +56,7 @@ func errx(err error) {
os.Exit(1)
}
var funcRegex = regexp.MustCompile(`^func\s+Test(.+)\s*\(`)
var funcRegex = regexp.MustCompile(`^func\s+BackendTest(.+)\s*\(`)
func findTestFunctions() (funcs []string) {
f, err := os.Open(*testFile)
@ -123,12 +119,7 @@ func main() {
f, err := os.Create(*outputFile)
errx(err)
data.Package = pkg + "_test"
if *prefix != "" {
data.PackagePrefix = *prefix
} else {
data.PackagePrefix = packageTestFunctionPrefix(pkg) + "Backend"
}
data.Package = pkg
data.Funcs = findTestFunctions()
generateOutput(f, data)

239
src/restic/backend/test/tests.go
View File

@ -1,3 +1,4 @@
// Package test contains a test suite for restic backends.
package test
import (
@ -19,112 +20,88 @@ import (
"restic/backend"
)
// CreateFn is a function that creates a temporary repository for the tests.
var CreateFn func() (restic.Backend, error)
// Suite implements a test suite for restic backends.
type Suite struct {
Config interface{}
// OpenFn is a function that opens a previously created temporary repository.
var OpenFn func() (restic.Backend, error)
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig func() (interface{}, error)
// CleanupFn removes temporary files and directories created during the tests.
var CleanupFn func() error
// CreateFn is a function that creates a temporary repository for the tests.
Create func(cfg interface{}) (restic.Backend, error)
var but restic.Backend // backendUnderTest
var butInitialized bool
// OpenFn is a function that opens a previously created temporary repository.
Open func(cfg interface{}) (restic.Backend, error)
func open(t testing.TB) restic.Backend {
if OpenFn == nil {
t.Fatal("OpenFn not set")
}
// CleanupFn removes data created during the tests.
Cleanup func(cfg interface{}) error
if CreateFn == nil {
t.Fatalf("CreateFn not set")
}
if !butInitialized {
be, err := CreateFn()
if err != nil {
t.Fatalf("Create returned unexpected error: %+v", err)
}
but = be
butInitialized = true
}
if but == nil {
var err error
but, err = OpenFn()
if err != nil {
t.Fatalf("Open returned unexpected error: %+v", err)
}
}
return but
// MinimalData instructs the tests to not use excessive data.
MinimalData bool
}
func close(t testing.TB) {
if but == nil {
t.Fatalf("trying to close non-existing backend")
}
err := but.Close()
// RunTests executes all defined tests as subtests of t.
func (s *Suite) RunTests(t *testing.T) {
var err error
s.Config, err = s.NewConfig()
if err != nil {
t.Fatalf("Close returned unexpected error: %+v", err)
t.Fatal(err)
}
but = nil
}
// test create/open functions first
be := s.create(t)
s.close(t, be)
// TestCreate creates a backend.
func TestCreate(t testing.TB) {
if CreateFn == nil {
t.Fatalf("CreateFn not set!")
for _, test := range testFunctions {
t.Run(test.Name, func(t *testing.T) {
test.Fn(t, s)
})
}
be, err := CreateFn()
if err != nil {
t.Fatalf("Create returned error: %+v", err)
if !test.TestCleanupTempDirs {
t.Logf("not cleaning up backend")
return
}
butInitialized = true
err = be.Close()
if err != nil {
t.Fatalf("Close returned error: %+v", err)
if err = s.Cleanup(s.Config); err != nil {
t.Fatal(err)
}
}
// TestOpen opens a previously created backend.
func TestOpen(t testing.TB) {
if OpenFn == nil {
t.Fatalf("OpenFn not set!")
}
be, err := OpenFn()
func (s *Suite) create(t testing.TB) restic.Backend {
be, err := s.Create(s.Config)
if err != nil {
t.Fatalf("Open returned error: %+v", err)
t.Fatal(err)
}
return be
}
err = be.Close()
func (s *Suite) open(t testing.TB) restic.Backend {
be, err := s.Open(s.Config)
if err != nil {
t.Fatalf("Close returned error: %+v", err)
t.Fatal(err)
}
return be
}
func (s *Suite) close(t testing.TB, be restic.Backend) {
err := be.Close()
if err != nil {
t.Fatal(err)
}
}
// TestCreateWithConfig tests that creating a backend in a location which already
// BackendTestCreateWithConfig tests that creating a backend in a location which already
// has a config file fails.
func TestCreateWithConfig(t testing.TB) {
if CreateFn == nil {
t.Fatalf("CreateFn not set")
}
b := open(t)
defer close(t)
func BackendTestCreateWithConfig(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
// save a config
store(t, b, restic.ConfigFile, []byte("test config"))
// now create the backend again, this must fail
_, err := CreateFn()
_, err := s.Create(s.Config)
if err == nil {
t.Fatalf("expected error not found for creating a backend with an existing config file")
}
@ -136,10 +113,10 @@ func TestCreateWithConfig(t testing.TB) {
}
}
// TestLocation tests that a location string is returned.
func TestLocation(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestLocation tests that a location string is returned.
func BackendTestLocation(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
l := b.Location()
if l == "" {
@ -147,10 +124,10 @@ func TestLocation(t testing.TB) {
}
}
// TestConfig saves and loads a config from the backend.
func TestConfig(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestConfig saves and loads a config from the backend.
func BackendTestConfig(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
var testString = "Config"
@ -180,10 +157,10 @@ func TestConfig(t testing.TB) {
}
}
// TestLoad tests the backend's Load function.
func TestLoad(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestLoad tests the backend's Load function.
func BackendTestLoad(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
_, err := b.Load(restic.Handle{}, 0, 0)
if err == nil {
@ -215,7 +192,12 @@ func TestLoad(t testing.TB) {
t.Fatalf("Load() returned a non-nil reader for negative offset!")
}
for i := 0; i < 50; i++ {
loadTests := 50
if s.MinimalData {
loadTests = 10
}
for i := 0; i < loadTests; i++ {
l := rand.Intn(length + 2000)
o := rand.Intn(length + 2000)
@ -245,31 +227,41 @@ func TestLoad(t testing.TB) {
buf, err := ioutil.ReadAll(rd)
if err != nil {
t.Errorf("Load(%d, %d) ReadAll() returned unexpected error: %+v", l, o, err)
rd.Close()
if err = rd.Close(); err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err)
}
continue
}
if l == 0 && len(buf) != len(d) {
t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, len(d), len(buf))
rd.Close()
if err = rd.Close(); err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err)
}
continue
}
if l > 0 && l <= len(d) && len(buf) != l {
t.Errorf("Load(%d, %d) wrong number of bytes read: want %d, got %d", l, o, l, len(buf))
rd.Close()
if err = rd.Close(); err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err)
}
continue
}
if l > len(d) && len(buf) != len(d) {
t.Errorf("Load(%d, %d) wrong number of bytes read for overlong read: want %d, got %d", l, o, l, len(buf))
rd.Close()
if err = rd.Close(); err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err)
}
continue
}
if !bytes.Equal(buf, d) {
t.Errorf("Load(%d, %d) returned wrong bytes", l, o)
rd.Close()
if err = rd.Close(); err != nil {
t.Errorf("Load(%d, %d) rd.Close() returned error: %+v", err)
}
continue
}
@ -293,13 +285,18 @@ func (ec errorCloser) Close() error {
return errors.New("forbidden method close was called")
}
// TestSave tests saving data in the backend.
func TestSave(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestSave tests saving data in the backend.
func BackendTestSave(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
var id restic.ID
for i := 0; i < 10; i++ {
saveTests := 10
if s.MinimalData {
saveTests = 2
}
for i := 0; i < saveTests; i++ {
length := rand.Intn(1<<23) + 200000
data := test.Random(23, length)
// use the first 32 byte as the ID
@ -388,10 +385,10 @@ var filenameTests = []struct {
},
}
// TestSaveFilenames tests saving data with various file names in the backend.
func TestSaveFilenames(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestSaveFilenames tests saving data with various file names in the backend.
func BackendTestSaveFilenames(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
for i, test := range filenameTests {
h := restic.Handle{Name: test.name, Type: restic.DataFile}
@ -437,10 +434,10 @@ func store(t testing.TB, b restic.Backend, tpe restic.FileType, data []byte) res
return h
}
// TestBackend tests all functions of the backend.
func TestBackend(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestBackend tests all functions of the backend.
func BackendTestBackend(t testing.TB, s *Suite) {
b := s.open(t)
defer s.close(t, b)
for _, tpe := range []restic.FileType{
restic.DataFile, restic.KeyFile, restic.LockFile,
@ -571,10 +568,14 @@ func TestBackend(t testing.TB) {
}
}
// TestDelete tests the Delete function.
func TestDelete(t testing.TB) {
b := open(t)
defer close(t)
// BackendTestDelete tests the Delete function.
func BackendTestDelete(t testing.TB, s *Suite) {
if !test.TestCleanupTempDirs {
t.Skipf("not removing backend, TestCleanupTempDirs is false")
}
b := s.open(t)
defer s.close(t, b)
be, ok := b.(restic.Deleter)
if !ok {
@ -586,21 +587,3 @@ func TestDelete(t testing.TB) {
t.Fatalf("error deleting backend: %+v", err)
}
}
// TestCleanup runs the cleanup function after all tests are run.
func TestCleanup(t testing.TB) {
if CleanupFn == nil {
t.Log("CleanupFn function not set")
return
}
if !test.TestCleanupTempDirs {
t.Logf("not cleaning up backend")
return
}
err := CleanupFn()
if err != nil {
t.Fatalf("Cleanup returned error: %+v", err)
}
}

75
src/restic/backend/test/tests_test.go
View File

@ -2,38 +2,59 @@ package test_test
import (
"restic"
"restic/errors"
"testing"
"restic/backend/mem"
"restic/backend/test"
)
var be restic.Backend
//go:generate go run generate_test_list.go
//go:generate go run ../test/generate_backend_tests.go
func init() {
test.CreateFn = func() (restic.Backend, error) {
if be != nil {
return nil, errors.New("temporary memory backend dir already exists")
}
be = mem.New()
return be, nil
}
test.OpenFn = func() (restic.Backend, error) {
if be == nil {
return nil, errors.New("repository not initialized")
}
return be, nil
}
test.CleanupFn = func() error {
be = nil
return nil
}
type memConfig struct {
be restic.Backend
}
func TestSuiteBackendMem(t *testing.T) {
suite := test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
return &memConfig{}, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be != nil {
ok, err := c.be.Test(restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if ok {
return nil, errors.New("config already exists")
}
}
c.be = mem.New()
return c.be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(cfg interface{}) (restic.Backend, error) {
c := cfg.(*memConfig)
if c.be == nil {
c.be = mem.New()
}
return c.be, nil
},
// CleanupFn removes data created during the tests.
Cleanup: func(cfg interface{}) error {
// no cleanup needed
return nil
},
}
suite.RunTests(t)
}

8
src/restic/backend/utils.go
View File

@ -14,8 +14,12 @@ func LoadAll(be restic.Backend, h restic.Handle) (buf []byte, err error) {
}
defer func() {
io.Copy(ioutil.Discard, rd)
e := rd.Close()
_, e := io.Copy(ioutil.Discard, rd)
if err == nil {
err = e
}
e = rd.Close()
if err == nil {
err = e
}

92
src/restic/debug/round_tripper_debug.go Normal file
View File

@ -0,0 +1,92 @@
// +build debug
package debug
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"restic/errors"
)
type eofDetectRoundTripper struct {
http.RoundTripper
}
type eofDetectReader struct {
eofSeen bool
rd io.ReadCloser
}
func (rd *eofDetectReader) Read(p []byte) (n int, err error) {
n, err = rd.rd.Read(p)
if err == io.EOF {
rd.eofSeen = true
}
return n, err
}
func (rd *eofDetectReader) Close() error {
if !rd.eofSeen {
buf, err := ioutil.ReadAll(rd)
msg := fmt.Sprintf("body not drained, %d bytes not read", len(buf))
if err != nil {
msg += fmt.Sprintf(", error: %v", err)
}
if len(buf) > 0 {
if len(buf) > 20 {
buf = append(buf[:20], []byte("...")...)
}
msg += fmt.Sprintf(", body: %q", buf)
}
fmt.Fprintln(os.Stderr, msg)
Log("%s: %+v", msg, errors.New("Close()"))
}
return rd.rd.Close()
}
func (tr eofDetectRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
res, err = tr.RoundTripper.RoundTrip(req)
res.Body = &eofDetectReader{rd: res.Body}
return res, err
}
type loggingRoundTripper struct {
http.RoundTripper
}
// RoundTripper returns a new http.RoundTripper which logs all requests (if
// debug is enabled). When debug is not enabled, upstream is returned.
func RoundTripper(upstream http.RoundTripper) http.RoundTripper {
return loggingRoundTripper{eofDetectRoundTripper{upstream}}
}
func (tr loggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, err error) {
trace, err := httputil.DumpRequestOut(req, false)
if err != nil {
Log("DumpRequestOut() error: %v\n", err)
} else {
Log("------------ HTTP REQUEST -----------\n%s", trace)
}
res, err = tr.RoundTripper.RoundTrip(req)
if err != nil {
Log("RoundTrip() returned error: %v", err)
}
if res != nil {
trace, err := httputil.DumpResponse(res, false)
if err != nil {
Log("DumpResponse() error: %v\n", err)
} else {
Log("------------ HTTP RESPONSE ----------\n%s", trace)
}
}
return res, err
}

11
src/restic/debug/round_tripper_release.go Normal file
View File

@ -0,0 +1,11 @@
// +build !debug
package debug
import "net/http"
// RoundTripper returns a new http.RoundTripper which logs all requests (if
// debug is enabled). When debug is not enabled, upstream is returned.
func RoundTripper(upstream http.RoundTripper) http.RoundTripper {
return upstream
}

21
src/restic/test/helpers.go
View File

@ -10,6 +10,7 @@ import (
"os/exec"
"path/filepath"
"reflect"
"restic/errors"
"runtime"
"testing"
@ -122,7 +123,7 @@ func SetupTarTestFixture(t testing.TB, outputDir, tarFile string) {
// Env creates a test environment and extracts the repository fixture.
// Returned is the repo path and a cleanup function.
func Env(t testing.TB, repoFixture string) (repodir string, cleanup func()) {
tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-env-")
OK(t, err)
fd, err := os.Open(repoFixture)
@ -151,7 +152,11 @@ func isFile(fi os.FileInfo) bool {
// This is mainly used for tests on Windows, which is unable to delete a file
// set read-only.
func ResetReadOnly(t testing.TB, dir string) {
OK(t, filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
err := filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if fi == nil {
return err
}
if fi.IsDir() {
return os.Chmod(path, 0777)
}
@ -161,14 +166,22 @@ func ResetReadOnly(t testing.TB, dir string) {
}
return nil
}))
})
if os.IsNotExist(errors.Cause(err)) {
err = nil
}
OK(t, err)
}
// RemoveAll recursively resets the read-only flag of all files and dirs and
// afterwards uses os.RemoveAll() to remove the path.
func RemoveAll(t testing.TB, path string) {
ResetReadOnly(t, path)
OK(t, os.RemoveAll(path))
err := os.RemoveAll(path)
if os.IsNotExist(errors.Cause(err)) {
err = nil
}
OK(t, err)
}
// TempDir returns a temporary directory that is removed when cleanup is