restic/internal/backend/sftp/sftp.go

502 lines
11 KiB
Go
Raw Normal View History

2015-03-28 10:50:23 +00:00
package sftp
2014-10-04 17:20:15 +00:00
import (
"bufio"
2017-06-03 15:39:57 +00:00
"context"
2014-10-04 17:20:15 +00:00
"fmt"
"io"
"os"
"os/exec"
"path"
"strings"
"time"
2014-10-04 17:20:15 +00:00
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/errors"
2017-07-24 15:42:25 +00:00
"github.com/restic/restic/internal/restic"
2017-07-23 12:21:03 +00:00
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/pkg/sftp"
2014-10-04 17:20:15 +00:00
)
2016-01-24 19:23:50 +00:00
// SFTP is a backend in a directory accessed via SFTP.
2014-10-04 17:20:15 +00:00
type SFTP struct {
c *sftp.Client
p string
2014-10-04 17:20:15 +00:00
cmd *exec.Cmd
result <-chan error
2017-04-10 20:40:24 +00:00
backend.Layout
2017-04-10 20:41:06 +00:00
Config
2014-10-04 17:20:15 +00:00
}
2016-08-31 20:51:35 +00:00
var _ restic.Backend = &SFTP{}
2017-04-10 20:40:24 +00:00
const defaultLayout = "default"
func startClient(preExec, postExec func(), program string, args ...string) (*SFTP, error) {
2017-04-03 19:05:42 +00:00
debug.Log("start client %v %v", program, args)
2014-10-04 17:20:15 +00:00
// Connect to a remote host and request the sftp subsystem via the 'ssh'
// command. This assumes that passwordless login is correctly configured.
cmd := exec.Command(program, args...)
// prefix the errors with the program name
stderr, err := cmd.StderrPipe()
if err != nil {
2016-08-29 19:54:50 +00:00
return nil, errors.Wrap(err, "cmd.StderrPipe")
}
go func() {
sc := bufio.NewScanner(stderr)
for sc.Scan() {
fmt.Fprintf(os.Stderr, "subprocess %v: %v\n", program, sc.Text())
}
}()
2014-10-04 17:20:15 +00:00
// get stdin and stdout
wr, err := cmd.StdinPipe()
if err != nil {
2016-08-29 19:54:50 +00:00
return nil, errors.Wrap(err, "cmd.StdinPipe")
2014-10-04 17:20:15 +00:00
}
rd, err := cmd.StdoutPipe()
if err != nil {
2016-08-29 19:54:50 +00:00
return nil, errors.Wrap(err, "cmd.StdoutPipe")
2014-10-04 17:20:15 +00:00
}
if preExec != nil {
preExec()
}
2014-10-04 17:20:15 +00:00
// start the process
if err := cmd.Start(); err != nil {
2016-08-29 19:54:50 +00:00
return nil, errors.Wrap(err, "cmd.Start")
2014-10-04 17:20:15 +00:00
}
if postExec != nil {
postExec()
}
// wait in a different goroutine
ch := make(chan error, 1)
go func() {
err := cmd.Wait()
2016-09-27 20:35:08 +00:00
debug.Log("ssh command exited, err %v", err)
2016-08-29 19:54:50 +00:00
ch <- errors.Wrap(err, "cmd.Wait")
}()
2014-10-04 17:20:15 +00:00
// open the SFTP session
client, err := sftp.NewClientPipe(rd, wr)
if err != nil {
return nil, errors.Errorf("unable to start the sftp session, error: %v", err)
2014-10-04 17:20:15 +00:00
}
return &SFTP{c: client, cmd: cmd, result: ch}, nil
2014-10-04 17:20:15 +00:00
}
// clientError returns an error if the client has exited. Otherwise, nil is
// returned immediately.
func (r *SFTP) clientError() error {
select {
case err := <-r.result:
2016-09-27 20:35:08 +00:00
debug.Log("client has exited with err %v", err)
return err
default:
}
return nil
}
2017-04-10 20:41:20 +00:00
// Open opens an sftp backend as described by the config by running
// "ssh" with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
func Open(cfg Config, preExec, postExec func()) (*SFTP, error) {
2017-04-10 20:41:20 +00:00
debug.Log("open backend with config %#v", cfg)
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
return nil, err
}
sftp, err := startClient(preExec, postExec, cmd, args...)
2014-10-04 17:20:15 +00:00
if err != nil {
2016-09-27 20:35:08 +00:00
debug.Log("unable to start program: %v", err)
2014-10-04 17:20:15 +00:00
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
2017-04-10 20:40:24 +00:00
if err != nil {
return nil, err
}
debug.Log("layout: %v\n", sftp.Layout)
2017-07-16 13:10:06 +00:00
if err := sftp.checkDataSubdirs(); err != nil {
debug.Log("checkDataSubdirs returned %v", err)
return nil, err
2017-07-02 13:58:03 +00:00
}
2017-04-10 20:41:20 +00:00
sftp.Config = cfg
sftp.p = cfg.Path
2014-10-04 17:20:15 +00:00
return sftp, nil
}
2017-07-16 13:10:06 +00:00
func (r *SFTP) checkDataSubdirs() error {
datadir := r.Dirname(restic.Handle{Type: restic.DataFile})
// check if all paths for data/ exist
entries, err := r.c.ReadDir(datadir)
if r.IsNotExist(err) {
return nil
}
2017-07-16 13:10:06 +00:00
if err != nil {
return err
}
subdirs := make(map[string]struct{}, len(entries))
for _, entry := range entries {
subdirs[entry.Name()] = struct{}{}
}
for i := 0; i < 256; i++ {
subdir := fmt.Sprintf("%02x", i)
if _, ok := subdirs[subdir]; !ok {
debug.Log("subdir %v is missing, creating", subdir)
err := r.mkdirAll(path.Join(datadir, subdir), backend.Modes.Dir)
if err != nil {
return err
}
}
}
return nil
}
func (r *SFTP) mkdirAllDataSubdirs() error {
for _, d := range r.Paths() {
err := r.mkdirAll(d, backend.Modes.Dir)
debug.Log("mkdirAll %v -> %v", d, err)
if err != nil {
return err
}
}
return nil
}
2017-04-10 20:41:20 +00:00
// Join combines path components with slashes (according to the sftp spec).
func (r *SFTP) Join(p ...string) string {
return path.Join(p...)
}
// ReadDir returns the entries for a directory.
func (r *SFTP) ReadDir(dir string) ([]os.FileInfo, error) {
return r.c.ReadDir(dir)
}
// IsNotExist returns true if the error is caused by a not existing file.
func (r *SFTP) IsNotExist(err error) bool {
if os.IsNotExist(err) {
return true
}
statusError, ok := err.(*sftp.StatusError)
if !ok {
return false
}
return statusError.Error() == `sftp: "No such file" (SSH_FX_NO_SUCH_FILE)`
}
2017-04-10 20:41:20 +00:00
func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
if cfg.Command != "" {
return SplitShellArgs(cfg.Command)
}
cmd = "ssh"
hostport := strings.Split(cfg.Host, ":")
2017-04-10 20:41:20 +00:00
args = []string{hostport[0]}
if len(hostport) > 1 {
args = append(args, "-p", hostport[1])
}
2015-12-28 17:22:19 +00:00
if cfg.User != "" {
args = append(args, "-l")
args = append(args, cfg.User)
}
args = append(args, "-s")
args = append(args, "sftp")
2017-04-10 20:41:20 +00:00
return cmd, args, nil
2015-12-28 17:22:19 +00:00
}
// Create creates an sftp backend as described by the config by running "ssh"
// with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
func Create(cfg Config, preExec, postExec func()) (*SFTP, error) {
2017-04-10 20:41:20 +00:00
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
return nil, err
2017-04-03 19:05:42 +00:00
}
sftp, err := startClient(preExec, postExec, cmd, args...)
2017-04-03 19:05:42 +00:00
if err != nil {
2017-04-10 20:41:20 +00:00
debug.Log("unable to start program: %v", err)
2017-04-03 19:05:42 +00:00
return nil, err
}
sftp.Layout, err = backend.ParseLayout(sftp, cfg.Layout, defaultLayout, cfg.Path)
2014-10-04 17:20:15 +00:00
if err != nil {
return nil, err
}
2015-05-04 18:39:45 +00:00
// test if config file already exists
2017-04-10 20:41:20 +00:00
_, err = sftp.c.Lstat(Join(cfg.Path, backend.Paths.Config))
2014-10-04 17:20:15 +00:00
if err == nil {
return nil, errors.New("config file already exists")
}
2017-04-19 16:56:01 +00:00
// create paths for data and refs
2017-07-16 13:10:06 +00:00
if err = sftp.mkdirAllDataSubdirs(); err != nil {
return nil, err
2014-10-04 17:20:15 +00:00
}
err = sftp.Close()
2014-10-04 17:20:15 +00:00
if err != nil {
2016-08-29 19:54:50 +00:00
return nil, errors.Wrap(err, "Close")
2014-10-04 17:20:15 +00:00
}
2014-10-07 21:19:26 +00:00
// open backend
return Open(cfg, preExec, postExec)
2015-12-28 17:22:19 +00:00
}
2014-10-04 17:20:15 +00:00
// Location returns this backend's location (the directory name).
func (r *SFTP) Location() string {
return r.p
}
func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
// check if directory already exists
fi, err := r.c.Lstat(dir)
if err == nil {
if fi.IsDir() {
return nil
}
return errors.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
}
// create parent directories
errMkdirAll := r.mkdirAll(path.Dir(dir), backend.Modes.Dir)
// create directory
errMkdir := r.c.Mkdir(dir)
// test if directory was created successfully
fi, err = r.c.Lstat(dir)
if err != nil {
// return previous errors
return errors.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
}
if !fi.IsDir() {
return errors.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
}
// set mode
return r.c.Chmod(dir, mode)
}
// Join joins the given paths and cleans them afterwards. This always uses
// forward slashes, which is required by sftp.
func Join(parts ...string) string {
return path.Clean(path.Join(parts...))
}
2016-01-24 00:15:35 +00:00
// Save stores data in the backend at the handle.
2017-06-03 15:39:57 +00:00
func (r *SFTP) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
2017-04-10 20:40:24 +00:00
debug.Log("Save %v", h)
if err := r.clientError(); err != nil {
return err
}
2016-01-24 00:15:35 +00:00
if err := h.Valid(); err != nil {
return err
}
2017-04-10 20:40:24 +00:00
filename := r.Filename(h)
// create new file
f, err := r.c.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY)
if r.IsNotExist(errors.Cause(err)) {
// create the locks dir, then try again
err = r.mkdirAll(r.Dirname(h), backend.Modes.Dir)
2017-04-10 20:40:24 +00:00
if err != nil {
return errors.Wrap(err, "MkdirAll")
2017-04-10 20:40:24 +00:00
}
2017-06-03 15:39:57 +00:00
return r.Save(ctx, h, rd)
2017-04-10 20:40:24 +00:00
}
if err != nil {
return errors.Wrap(err, "OpenFile")
2017-04-10 20:40:24 +00:00
}
// save data
_, err = io.Copy(f, rd)
2017-04-10 20:40:24 +00:00
if err != nil {
2017-05-11 19:53:57 +00:00
_ = f.Close()
return errors.Wrap(err, "Write")
}
err = f.Close()
if err != nil {
return errors.Wrap(err, "Close")
2017-04-10 20:40:24 +00:00
}
return errors.Wrap(r.c.Chmod(filename, backend.Modes.File), "Chmod")
2016-01-24 00:15:35 +00:00
}
2017-01-23 17:11:10 +00:00
// Load returns a reader that yields the contents of the file at h at the
2017-01-22 21:01:12 +00:00
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
2017-06-03 15:39:57 +00:00
func (r *SFTP) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
2017-01-23 17:11:10 +00:00
debug.Log("Load %v, length %v, offset %v", h, length, offset)
2017-01-22 21:01:12 +00:00
if err := h.Valid(); err != nil {
return nil, err
}
if offset < 0 {
return nil, errors.New("offset is negative")
}
f, err := r.c.Open(r.Filename(h))
2017-01-22 21:01:12 +00:00
if err != nil {
return nil, err
}
if offset > 0 {
_, err = f.Seek(offset, 0)
if err != nil {
_ = f.Close()
2017-01-22 21:01:12 +00:00
return nil, err
}
}
if length > 0 {
return backend.LimitReadCloser(f, int64(length)), nil
}
return f, nil
}
2016-01-23 22:27:58 +00:00
// Stat returns information about a blob.
2017-06-03 15:39:57 +00:00
func (r *SFTP) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
debug.Log("Stat(%v)", h)
if err := r.clientError(); err != nil {
2016-08-31 20:51:35 +00:00
return restic.FileInfo{}, err
}
2016-01-23 22:27:58 +00:00
if err := h.Valid(); err != nil {
2016-08-31 20:51:35 +00:00
return restic.FileInfo{}, err
2016-01-23 22:27:58 +00:00
}
fi, err := r.c.Lstat(r.Filename(h))
2016-01-23 22:27:58 +00:00
if err != nil {
2016-08-31 20:51:35 +00:00
return restic.FileInfo{}, errors.Wrap(err, "Lstat")
2016-01-23 22:27:58 +00:00
}
2016-08-31 20:51:35 +00:00
return restic.FileInfo{Size: fi.Size()}, nil
2016-01-23 22:27:58 +00:00
}
2015-03-28 10:50:23 +00:00
// Test returns true if a blob of the given type and name exists in the backend.
2017-06-03 15:39:57 +00:00
func (r *SFTP) Test(ctx context.Context, h restic.Handle) (bool, error) {
debug.Log("Test(%v)", h)
if err := r.clientError(); err != nil {
return false, err
}
_, err := r.c.Lstat(r.Filename(h))
if os.IsNotExist(errors.Cause(err)) {
return false, nil
}
if err != nil {
2016-08-29 19:54:50 +00:00
return false, errors.Wrap(err, "Lstat")
2014-10-04 17:20:15 +00:00
}
return true, nil
2014-10-04 17:20:15 +00:00
}
2015-03-28 10:50:23 +00:00
// Remove removes the content stored at name.
2017-06-03 15:39:57 +00:00
func (r *SFTP) Remove(ctx context.Context, h restic.Handle) error {
debug.Log("Remove(%v)", h)
if err := r.clientError(); err != nil {
return err
}
return r.c.Remove(r.Filename(h))
2014-10-04 17:20:15 +00:00
}
2015-03-28 10:50:23 +00:00
// List returns a channel that yields all names of blobs of type t. A
2015-06-28 07:44:06 +00:00
// goroutine is started for this. If the channel done is closed, sending
2015-03-28 10:50:23 +00:00
// stops.
2017-06-03 15:39:57 +00:00
func (r *SFTP) List(ctx context.Context, t restic.FileType) <-chan string {
2017-04-10 21:31:13 +00:00
debug.Log("List %v", t)
2015-03-28 10:50:23 +00:00
ch := make(chan string)
2015-03-28 10:50:23 +00:00
go func() {
defer close(ch)
2017-04-10 21:31:13 +00:00
walker := r.c.Walk(r.Basedir(t))
for walker.Step() {
if walker.Err() != nil {
continue
2015-03-28 10:50:23 +00:00
}
2017-04-10 21:31:13 +00:00
if !walker.Stat().Mode().IsRegular() {
continue
2015-03-28 10:50:23 +00:00
}
2017-04-10 21:31:13 +00:00
select {
2017-05-28 08:41:57 +00:00
case ch <- path.Base(walker.Path()):
2017-06-03 15:39:57 +00:00
case <-ctx.Done():
2015-03-28 10:50:23 +00:00
return
}
2014-10-04 17:20:15 +00:00
}
2015-03-28 10:50:23 +00:00
}()
return ch
2014-10-04 17:20:15 +00:00
}
var closeTimeout = 2 * time.Second
2014-10-04 17:20:15 +00:00
// Close closes the sftp connection and terminates the underlying command.
2016-01-24 19:23:50 +00:00
func (r *SFTP) Close() error {
2016-09-27 20:35:08 +00:00
debug.Log("")
2016-01-24 19:23:50 +00:00
if r == nil {
return nil
}
err := r.c.Close()
2016-09-27 20:35:08 +00:00
debug.Log("Close returned error %v", err)
// wait for closeTimeout before killing the process
select {
case err := <-r.result:
return err
case <-time.After(closeTimeout):
}
2016-01-24 19:23:50 +00:00
if err := r.cmd.Process.Kill(); err != nil {
return err
}
// get the error, but ignore it
<-r.result
return nil
2014-10-04 17:20:15 +00:00
}