From 6e775d3787d4614e2805f15e46d62f1ed67195e6 Mon Sep 17 00:00:00 2001
From: Adam Eijdenberg <adam.eijdenberg@defence.gov.au>
Date: Mon, 29 Jan 2024 14:23:56 +1100
Subject: [PATCH] Enhancement: option to send HTTP over unix socket
add tests for unix socket connection
switch HTTP rest-server test to use any free port
allow rest-server test graceful shutdown opportunity
---
doc/030_preparing_a_new_repo.rst | 7 +-
go.mod | 1 +
go.sum | 13 ++
internal/backend/http_transport.go | 3 +
internal/backend/rest/config_test.go | 7 +
internal/backend/rest/rest_test.go | 162 ++++++++++++++++++------
internal/backend/rest/rest_unix_test.go | 30 +++++
7 files changed, 182 insertions(+), 41 deletions(-)
create mode 100644 internal/backend/rest/rest_unix_test.go
diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst
index 8661f5904..0c50b65be 100644
--- a/doc/030_preparing_a_new_repo.rst
+++ b/doc/030_preparing_a_new_repo.rst
@@ -201,15 +201,16 @@ scheme like this:
$ restic -r rest:http://host:8000/ init
Depending on your REST server setup, you can use HTTPS protocol,
-password protection, multiple repositories or any combination of
-those features. The TCP/IP port is also configurable. Here
-are some more examples:
+unix socket, password protection, multiple repositories or any
+combination of those features. The TCP/IP port is also configurable.
+Here are some more examples:
.. code-block:: console
$ restic -r rest:https://host:8000/ init
$ restic -r rest:https://user:pass@host:8000/ init
$ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init
+ $ restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ init
The server username and password can be specified using environment
variables as well:
diff --git a/go.mod b/go.mod
index 6e546974e..7121cdac4 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
github.com/minio/minio-go/v7 v7.0.66
github.com/minio/sha256-simd v1.0.1
github.com/ncw/swift/v2 v2.0.2
+ github.com/peterbourgon/unixtransport v0.0.4
github.com/pkg/errors v0.9.1
github.com/pkg/profile v1.7.0
github.com/pkg/sftp v1.13.6
diff --git a/go.sum b/go.sum
index 668d6a339..1cd51cbac 100644
--- a/go.sum
+++ b/go.sum
@@ -128,6 +128,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
@@ -141,6 +142,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
+github.com/peterbourgon/unixtransport v0.0.4 h1:UTF0FxXCAglvoZz9jaGPYjEg52DjBLDYGMJvJni6Tfw=
+github.com/peterbourgon/unixtransport v0.0.4/go.mod h1:o8aUkOCa8W/BIXpi15uKvbSabjtBh0JhSOJGSfoOhAU=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -210,6 +216,7 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -221,6 +228,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -231,6 +239,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -246,12 +255,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -271,6 +282,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -315,6 +327,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/backend/http_transport.go b/internal/backend/http_transport.go
index 9ee1c91f1..19b20dc6a 100644
--- a/internal/backend/http_transport.go
+++ b/internal/backend/http_transport.go
@@ -10,6 +10,7 @@ import (
"strings"
"time"
+ "github.com/peterbourgon/unixtransport"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
)
@@ -82,6 +83,8 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
TLSClientConfig: &tls.Config{},
}
+ unixtransport.Register(tr)
+
if opts.InsecureTLS {
tr.TLSClientConfig.InsecureSkipVerify = true
}
diff --git a/internal/backend/rest/config_test.go b/internal/backend/rest/config_test.go
index 23ea9095b..13a1ebb13 100644
--- a/internal/backend/rest/config_test.go
+++ b/internal/backend/rest/config_test.go
@@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{
Connections: 5,
},
},
+ {
+ S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/",
+ Cfg: Config{
+ URL: parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"),
+ Connections: 5,
+ },
+ },
}
func TestParseConfig(t *testing.T) {
diff --git a/internal/backend/rest/rest_test.go b/internal/backend/rest/rest_test.go
index 6a5b4f8a5..93b9a103e 100644
--- a/internal/backend/rest/rest_test.go
+++ b/internal/backend/rest/rest_test.go
@@ -1,11 +1,18 @@
+//go:build go1.20
+// +build go1.20
+
package rest_test
import (
+ "bufio"
"context"
- "net"
+ "fmt"
"net/url"
"os"
"os/exec"
+ "regexp"
+ "strings"
+ "syscall"
"testing"
"time"
@@ -14,54 +21,133 @@ import (
rtest "github.com/restic/restic/internal/test"
)
-func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) {
+var (
+ serverStartedRE = regexp.MustCompile("^start server on (.*)$")
+)
+
+func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) {
srv, err := exec.LookPath("rest-server")
if err != nil {
t.Skip(err)
}
- cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir)
+ // create our own context, so that our cleanup can cancel and wait for completion
+ // this will ensure any open ports, open unix sockets etc are properly closed
+ processCtx, cancel := context.WithCancel(ctx)
+ cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr)
+
+ // this cancel func is called by when the process context is done
+ cmd.Cancel = func() error {
+ // we execute in a Go-routine as we know the caller will
+ // be waiting on a .Wait() regardless
+ go func() {
+ // try to send a graceful termination signal
+ if cmd.Process.Signal(syscall.SIGTERM) == nil {
+ // if we succeed, then wait a few seconds
+ time.Sleep(2 * time.Second)
+ }
+ // and then make sure it's killed either way, ignoring any error code
+ _ = cmd.Process.Kill()
+ }()
+ return nil
+ }
+
+ // this is the cleanup function that we return the caller,
+ // which will cancel our process context, and then wait for it to finish
+ cleanup := func() {
+ cancel()
+ _ = cmd.Wait()
+ }
+
+ // but in-case we don't finish this method, e.g. by calling t.Fatal()
+ // we also defer a call to clean it up ourselves, guarded by a flag to
+ // indicate that we returned the function to the caller to deal with.
+ callerWillCleanUp := false
+ defer func() {
+ if !callerWillCleanUp {
+ cleanup()
+ }
+ }()
+
+ // send stdout to our std out
cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stdout
- if err := cmd.Start(); err != nil {
- t.Fatal(err)
- }
- // wait until the TCP port is reachable
- var success bool
- for i := 0; i < 10; i++ {
- time.Sleep(200 * time.Millisecond)
-
- c, err := net.Dial("tcp", "localhost:8000")
- if err != nil {
- continue
- }
-
- success = true
- if err := c.Close(); err != nil {
- t.Fatal(err)
- }
- }
-
- if !success {
- t.Fatal("unable to connect to rest server")
- return nil, nil
- }
-
- url, err := url.Parse("http://localhost:8000/restic-test/")
+ // capture stderr with a pipe, as we want to examine this output
+ // to determine when the server is started and listening.
+ cmdErr, err := cmd.StderrPipe()
if err != nil {
t.Fatal(err)
}
- cleanup := func() {
- if err := cmd.Process.Kill(); err != nil {
- t.Fatal(err)
- }
-
- // ignore errors, we've killed the process
- _ = cmd.Wait()
+ // start the rest-server
+ if err := cmd.Start(); err != nil {
+ t.Fatal(err)
}
+ // create a channel to receive the actual listen address on
+ listenAddrCh := make(chan string)
+ go func() {
+ defer close(listenAddrCh)
+ matched := false
+ br := bufio.NewReader(cmdErr)
+ for {
+ line, err := br.ReadString('\n')
+ if err != nil {
+ // we ignore errors, as code that relies on this
+ // will happily fail via timeout and empty closed
+ // channel.
+ return
+ }
+
+ line = strings.Trim(line, "\r\n")
+ if !matched {
+ // look for the server started message, and return the address
+ // that it's listening on
+ matchedServerListen := serverStartedRE.FindSubmatch([]byte(line))
+ if len(matchedServerListen) == 2 {
+ listenAddrCh <- string(matchedServerListen[1])
+ matched = true
+ }
+ }
+ fmt.Fprintln(os.Stdout, line) // print all output to console
+ }
+ }()
+
+ // wait for us to get an address,
+ // or the parent context to cancel,
+ // or for us to timeout
+ var actualListenAddr string
+ select {
+ case <-processCtx.Done():
+ t.Fatal(context.Canceled)
+ case <-time.NewTimer(2 * time.Second).C:
+ t.Fatal(context.DeadlineExceeded)
+ case a, ok := <-listenAddrCh:
+ if !ok {
+ t.Fatal(context.Canceled)
+ }
+ actualListenAddr = a
+ }
+
+ // this translate the address that the server is listening on
+ // to a URL suitable for us to connect to
+ var addrToConnectTo string
+ if strings.HasPrefix(reqListenAddr, "unix:") {
+ addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr)
+ } else {
+ // while we may listen on 0.0.0.0, we connect to localhost
+ addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1))
+ }
+
+ // parse to a URL
+ url, err := url.Parse(addrToConnectTo)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // indicate that we've completed successfully, and that the caller
+ // is responsible for calling cleanup
+ callerWillCleanUp = true
return url, cleanup
}
@@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) {
defer cancel()
dir := rtest.TempDir(t)
- serverURL, cleanup := runRESTServer(ctx, t, dir)
+ serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
defer cleanup()
newTestSuite(serverURL, false).RunTests(t)
@@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) {
defer cancel()
dir := rtest.TempDir(t)
- serverURL, cleanup := runRESTServer(ctx, t, dir)
+ serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
defer cleanup()
newTestSuite(serverURL, false).RunBenchmarks(t)
diff --git a/internal/backend/rest/rest_unix_test.go b/internal/backend/rest/rest_unix_test.go
new file mode 100644
index 000000000..85ef7a73d
--- /dev/null
+++ b/internal/backend/rest/rest_unix_test.go
@@ -0,0 +1,30 @@
+//go:build !windows && go1.20
+// +build !windows,go1.20
+
+package rest_test
+
+import (
+ "context"
+ "fmt"
+ "path"
+ "testing"
+
+ rtest "github.com/restic/restic/internal/test"
+)
+
+func TestBackendRESTWithUnixSocket(t *testing.T) {
+ defer func() {
+ if t.Skipped() {
+ rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+ }
+ }()
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ dir := rtest.TempDir(t)
+ serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock")))
+ defer cleanup()
+
+ newTestSuite(serverURL, false).RunTests(t)
+}