restic/internal/backend/s3/s3_test.go

329 lines
7.0 KiB
Go
Raw Normal View History

2016-01-23 17:46:04 +00:00
package s3_test
2016-01-02 13:38:45 +00:00
2016-01-23 17:46:04 +00:00
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
2017-05-11 20:48:46 +00:00
"fmt"
"io"
"net"
"net/http"
2016-01-23 17:46:04 +00:00
"os"
"os/exec"
"path/filepath"
"testing"
"time"
"github.com/restic/restic/internal/backend"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/backend/s3"
"github.com/restic/restic/internal/backend/test"
2017-07-24 15:42:25 +00:00
"github.com/restic/restic/internal/restic"
2017-10-02 13:06:39 +00:00
rtest "github.com/restic/restic/internal/test"
2016-01-23 17:46:04 +00:00
)
2016-01-02 13:38:45 +00:00
func mkdir(t testing.TB, dir string) {
err := os.MkdirAll(dir, 0700)
if err != nil {
t.Fatal(err)
2016-01-23 17:46:04 +00:00
}
}
2016-01-23 17:46:04 +00:00
func runMinio(ctx context.Context, t testing.TB, dir, key, secret string) func() {
mkdir(t, filepath.Join(dir, "config"))
mkdir(t, filepath.Join(dir, "root"))
cmd := exec.CommandContext(ctx, "minio",
"server",
"--address", "127.0.0.1:9000",
"--config-dir", filepath.Join(dir, "config"),
filepath.Join(dir, "root"))
cmd.Env = append(os.Environ(),
"MINIO_ACCESS_KEY="+key,
"MINIO_SECRET_KEY="+secret,
)
cmd.Stderr = os.Stderr
err := cmd.Start()
2016-01-23 17:46:04 +00:00
if err != nil {
t.Fatal(err)
2016-01-23 17:46:04 +00:00
}
// wait until the TCP port is reachable
var success bool
for i := 0; i < 100; i++ {
time.Sleep(200 * time.Millisecond)
c, err := net.Dial("tcp", "localhost:9000")
if err == nil {
success = true
if err := c.Close(); err != nil {
t.Fatal(err)
}
break
}
2016-01-23 17:46:04 +00:00
}
if !success {
t.Fatal("unable to connect to minio server")
return nil
2016-01-23 17:46:04 +00:00
}
return func() {
err = cmd.Process.Kill()
2016-01-23 17:46:04 +00:00
if err != nil {
t.Fatal(err)
2016-01-23 17:46:04 +00:00
}
// ignore errors, we've killed the process
_ = cmd.Wait()
}
}
2016-01-23 17:46:04 +00:00
2017-05-13 19:44:19 +00:00
func newRandomCredentials(t testing.TB) (key, secret string) {
buf := make([]byte, 10)
_, err := io.ReadFull(rand.Reader, buf)
if err != nil {
t.Fatal(err)
}
key = hex.EncodeToString(buf)
2016-01-23 17:46:04 +00:00
_, err = io.ReadFull(rand.Reader, buf)
if err != nil {
t.Fatal(err)
}
secret = hex.EncodeToString(buf)
2016-01-23 17:46:04 +00:00
return key, secret
}
2017-05-13 19:44:19 +00:00
type MinioTestConfig struct {
s3.Config
2017-05-13 19:44:19 +00:00
tempdir string
removeTempdir func()
stopServer func()
}
2016-01-23 17:46:04 +00:00
func createS3(t testing.TB, cfg MinioTestConfig, tr http.RoundTripper) (be restic.Backend, err error) {
for i := 0; i < 10; i++ {
be, err = s3.Create(cfg.Config, tr)
if err != nil {
t.Logf("s3 open: try %d: error %v", i, err)
time.Sleep(500 * time.Millisecond)
continue
}
break
}
return be, err
}
2017-05-13 19:44:19 +00:00
func newMinioTestSuite(ctx context.Context, t testing.TB) *test.Suite {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
2017-05-13 19:44:19 +00:00
return &test.Suite{
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
2017-05-13 19:44:19 +00:00
cfg := MinioTestConfig{}
2017-10-02 13:06:39 +00:00
cfg.tempdir, cfg.removeTempdir = rtest.TempDir(t)
2017-05-13 19:44:19 +00:00
key, secret := newRandomCredentials(t)
cfg.stopServer = runMinio(ctx, t, cfg.tempdir, key, secret)
2017-06-05 22:17:39 +00:00
cfg.Config = s3.NewConfig()
cfg.Config.Endpoint = "localhost:9000"
cfg.Config.Bucket = "restictestbucket"
cfg.Config.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
cfg.Config.UseHTTP = true
cfg.Config.KeyID = key
cfg.Config.Secret = secret
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
2017-05-13 19:44:19 +00:00
cfg := config.(MinioTestConfig)
be, err := createS3(t, cfg, tr)
if err != nil {
return nil, err
}
2017-06-03 15:39:57 +00:00
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
2017-05-13 19:44:19 +00:00
cfg := config.(MinioTestConfig)
return s3.Open(cfg.Config, tr)
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
2017-05-13 19:44:19 +00:00
cfg := config.(MinioTestConfig)
if cfg.stopServer != nil {
cfg.stopServer()
}
if cfg.removeTempdir != nil {
cfg.removeTempdir()
}
return nil
},
}
2016-01-02 13:38:45 +00:00
}
2017-05-11 20:38:25 +00:00
2017-05-13 19:44:19 +00:00
func TestBackendMinio(t *testing.T) {
defer func() {
if t.Skipped() {
2017-10-02 13:06:39 +00:00
rtest.SkipDisallowed(t, "restic/backend/s3.TestBackendMinio")
}
}()
2017-05-13 19:44:19 +00:00
// try to find a minio binary
_, err := exec.LookPath("minio")
if err != nil {
t.Skip(err)
return
2017-05-11 20:38:25 +00:00
}
2017-05-13 19:44:19 +00:00
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
newMinioTestSuite(ctx, t).RunTests(t)
}
func BenchmarkBackendMinio(t *testing.B) {
// try to find a minio binary
_, err := exec.LookPath("minio")
if err != nil {
t.Skip(err)
return
2017-05-11 20:38:25 +00:00
}
2017-05-13 19:44:19 +00:00
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
newMinioTestSuite(ctx, t).RunBenchmarks(t)
}
func newS3TestSuite(t testing.TB) *test.Suite {
tr, err := backend.Transport(backend.TransportOptions{})
if err != nil {
t.Fatalf("cannot create transport for tests: %v", err)
}
2017-05-13 19:44:19 +00:00
return &test.Suite{
2017-05-11 20:38:25 +00:00
// do not use excessive data
MinimalData: true,
// NewConfig returns a config for a new temporary backend that will be used in tests.
NewConfig: func() (interface{}, error) {
s3cfg, err := s3.ParseConfig(os.Getenv("RESTIC_TEST_S3_REPOSITORY"))
if err != nil {
return nil, err
}
cfg := s3cfg.(s3.Config)
cfg.KeyID = os.Getenv("RESTIC_TEST_S3_KEY")
cfg.Secret = os.Getenv("RESTIC_TEST_S3_SECRET")
2017-05-11 20:48:46 +00:00
cfg.Prefix = fmt.Sprintf("test-%d", time.Now().UnixNano())
2017-05-11 20:38:25 +00:00
return cfg, nil
},
// CreateFn is a function that creates a temporary repository for the tests.
Create: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
be, err := s3.Create(cfg, tr)
2017-05-11 20:38:25 +00:00
if err != nil {
return nil, err
}
2017-06-03 15:39:57 +00:00
exists, err := be.Test(context.TODO(), restic.Handle{Type: restic.ConfigFile})
2017-05-11 20:38:25 +00:00
if err != nil {
return nil, err
}
if exists {
return nil, errors.New("config already exists")
}
return be, nil
},
// OpenFn is a function that opens a previously created temporary repository.
Open: func(config interface{}) (restic.Backend, error) {
cfg := config.(s3.Config)
return s3.Open(cfg, tr)
2017-05-11 20:38:25 +00:00
},
// CleanupFn removes data created during the tests.
Cleanup: func(config interface{}) error {
cfg := config.(s3.Config)
be, err := s3.Open(cfg, tr)
2017-05-11 20:38:25 +00:00
if err != nil {
return err
}
2017-10-14 13:56:38 +00:00
return be.Delete(context.TODO())
2017-05-11 20:38:25 +00:00
},
}
2017-05-13 19:44:19 +00:00
}
func TestBackendS3(t *testing.T) {
defer func() {
if t.Skipped() {
2017-10-02 13:06:39 +00:00
rtest.SkipDisallowed(t, "restic/backend/s3.TestBackendS3")
2017-05-13 19:44:19 +00:00
}
}()
vars := []string{
"RESTIC_TEST_S3_KEY",
"RESTIC_TEST_S3_SECRET",
"RESTIC_TEST_S3_REPOSITORY",
}
for _, v := range vars {
if os.Getenv(v) == "" {
t.Skipf("environment variable %v not set", v)
return
}
}
t.Logf("run tests")
newS3TestSuite(t).RunTests(t)
}
func BenchmarkBackendS3(t *testing.B) {
vars := []string{
"RESTIC_TEST_S3_KEY",
"RESTIC_TEST_S3_SECRET",
"RESTIC_TEST_S3_REPOSITORY",
}
for _, v := range vars {
if os.Getenv(v) == "" {
t.Skipf("environment variable %v not set", v)
return
}
}
2017-05-11 20:38:25 +00:00
t.Logf("run tests")
2017-05-13 19:44:19 +00:00
newS3TestSuite(t).RunBenchmarks(t)
2017-05-11 20:38:25 +00:00
}