mirror
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Releases Wiki Activity

repository: move backend.LoadAll to repository.LoadRaw 

LoadRaw also includes improved context cancellation handling similar to the
implementation in repository.LoadUnpacked.

The removed cache backend test will be added again later on.
This commit is contained in:
Michael Eischer 2024-05-09 18:59:29 +02:00
parent 47232bf8b0
commit 1d6d3656b0
11 changed files with 113 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/restic/cmd_cat.go
View File

@ -7,7 +7,6 @@ import (
"github.com/spf13/cobra"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -146,9 +145,9 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
return nil
case "pack":
h := backend.Handle{Type: restic.PackFile, Name: id.String()}
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// allow returning broken pack files
if buf == nil {
return err
}

5
cmd/restic/cmd_debug.go
View File

@ -492,8 +492,9 @@ func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repo
}
Printf(" file size is %v\n", fi.Size)
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
if err != nil {
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
// also process damaged pack files
if buf == nil {
return err
}
gotID := restic.Hash(buf)

21
internal/backend/test/tests.go
View File

@ -36,6 +36,19 @@ func beTest(ctx context.Context, be backend.Backend, h backend.Handle) (bool, er
return err == nil, err
}
func LoadAll(ctx context.Context, be backend.Backend, h backend.Handle) ([]byte, error) {
var buf []byte
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
var err error
buf, err = io.ReadAll(rd)
return err
})
if err != nil {
return nil, err
}
return buf, nil
}
// TestStripPasswordCall tests that the StripPassword method of a factory can be called without crashing.
// It does not verify whether passwords are removed correctly
func (s *Suite[C]) TestStripPasswordCall(_ *testing.T) {
@ -94,7 +107,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
var testString = "Config"
// create config and read it back
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.ConfigFile})
_, err := LoadAll(context.TODO(), b, backend.Handle{Type: backend.ConfigFile})
if err == nil {
t.Fatalf("did not get expected error for non-existing config")
}
@ -110,7 +123,7 @@ func (s *Suite[C]) TestConfig(t *testing.T) {
// same config
for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} {
h := backend.Handle{Type: backend.ConfigFile, Name: name}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
buf, err := LoadAll(context.TODO(), b, h)
if err != nil {
t.Fatalf("unable to read config with name %q: %+v", name, err)
}
@ -519,7 +532,7 @@ func (s *Suite[C]) TestSave(t *testing.T) {
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
test.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
buf, err := LoadAll(context.TODO(), b, h)
test.OK(t, err)
if len(buf) != len(data) {
t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf))
@ -821,7 +834,7 @@ func (s *Suite[C]) TestBackend(t *testing.T) {
// test Load()
h := backend.Handle{Type: tpe, Name: ts.id}
buf, err := backend.LoadAll(context.TODO(), nil, b, h)
buf, err := LoadAll(context.TODO(), b, h)
test.OK(t, err)
test.Equals(t, ts.data, string(buf))

64
internal/backend/utils.go
View File

@ -1,64 +0,0 @@
package backend
import (
"bytes"
"context"
"encoding/hex"
"fmt"
"io"
"github.com/minio/sha256-simd"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/errors"
)
func verifyContentMatchesName(s string, data []byte) (bool, error) {
if len(s) != hex.EncodedLen(sha256.Size) {
return false, fmt.Errorf("invalid length for ID: %q", s)
}
b, err := hex.DecodeString(s)
if err != nil {
return false, fmt.Errorf("invalid ID: %s", err)
}
var id [sha256.Size]byte
copy(id[:], b)
hashed := sha256.Sum256(data)
return id == hashed, nil
}
// LoadAll reads all data stored in the backend for the handle into the given
// buffer, which is truncated. If the buffer is not large enough or nil, a new
// one is allocated.
func LoadAll(ctx context.Context, buf []byte, be Backend, h Handle) ([]byte, error) {
retriedInvalidData := false
err := be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
// retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend. Return the data
// to the caller in that case to let it decide what to do with the data.
if !retriedInvalidData && h.Type != ConfigFile {
if matches, err := verifyContentMatchesName(h.Name, buf); err == nil && !matches {
debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true
return errors.Errorf("loadAll(%v): invalid data returned", h)
}
}
return nil
})
if err != nil {
return nil, err
}
return buf, nil
}

40
internal/cache/backend_test.go vendored
View File

@ -12,12 +12,13 @@ import (
"github.com/pkg/errors"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
backendtest "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
)
func loadAndCompare(t testing.TB, be backend.Backend, h backend.Handle, data []byte) {
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
buf, err := backendtest.LoadAll(context.TODO(), be, h)
if err != nil {
t.Fatal(err)
}
@ -140,7 +141,7 @@ func TestErrorBackend(t *testing.T) {
loadTest := func(wg *sync.WaitGroup, be backend.Backend) {
defer wg.Done()
buf, err := backend.LoadAll(context.TODO(), nil, be, h)
buf, err := backendtest.LoadAll(context.TODO(), be, h)
if err == testErr {
return
}
@ -165,38 +166,3 @@ func TestErrorBackend(t *testing.T) {
wg.Wait()
}
func TestBackendRemoveBroken(t *testing.T) {
be := mem.New()
c := TestNewCache(t)
h, data := randomData(5234142)
// save directly in backend
save(t, be, h, data)
// prime cache with broken copy
broken := append([]byte{}, data...)
broken[0] ^= 0xff
err := c.Save(h, bytes.NewReader(broken))
test.OK(t, err)
// loadall retries if broken data was returned
buf, err := backend.LoadAll(context.TODO(), nil, c.Wrap(be), h)
test.OK(t, err)
if !bytes.Equal(buf, data) {
t.Fatalf("wrong data returned")
}
// check that the cache now contains the correct data
rd, err := c.load(h, 0, 0)
defer func() {
_ = rd.Close()
}()
test.OK(t, err)
cached, err := io.ReadAll(rd)
test.OK(t, err)
if !bytes.Equal(cached, data) {
t.Fatalf("wrong data cache")
}
}

3
internal/repository/key.go
View File

@ -178,8 +178,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int,
// LoadKey loads a key from the backend.
func LoadKey(ctx context.Context, s *Repository, id restic.ID) (k *Key, err error) {
h := backend.Handle{Type: restic.KeyFile, Name: id.String()}
data, err := backend.LoadAll(ctx, nil, s.be, h)
data, err := s.LoadRaw(ctx, restic.KeyFile, id)
if err != nil {
return nil, err
}

63
internal/repository/raw.go Normal file
View File

@ -0,0 +1,63 @@
package repository
import (
"bytes"
"context"
"fmt"
"io"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// LoadRaw reads all data stored in the backend for the file with id and filetype t.
// If the backend returns data that does not match the id, then the buffer is returned
// along with an error that is a restic.ErrInvalidData error.
func (r *Repository) LoadRaw(ctx context.Context, t restic.FileType, id restic.ID) (buf []byte, err error) {
h := backend.Handle{Type: t, Name: id.String()}
ctx, cancel := context.WithCancel(ctx)
var dataErr error
retriedInvalidData := false
err = r.be.Load(ctx, h, 0, 0, func(rd io.Reader) error {
// make sure this is idempotent, in case an error occurs this function may be called multiple times!
wr := bytes.NewBuffer(buf[:0])
_, cerr := io.Copy(wr, rd)
if cerr != nil {
return cerr
}
buf = wr.Bytes()
// retry loading damaged data only once. If a file fails to download correctly
// the second time, then it is likely corrupted at the backend.
if h.Type != backend.ConfigFile {
if id != restic.Hash(buf) {
if !retriedInvalidData {
debug.Log("retry loading broken blob %v", h)
retriedInvalidData = true
} else {
// with a canceled context there is not guarantee which error will
// be returned by `be.Load`.
dataErr = fmt.Errorf("loadAll(%v): %w", h, restic.ErrInvalidData)
cancel()
}
return restic.ErrInvalidData
}
}
return nil
})
// Return corrupted data to the caller if it is still broken the second time to
// let the caller decide what to do with the data.
if dataErr != nil {
return buf, dataErr
}
if err != nil {
return nil, err
}
return buf, nil
}

86
internal/backend/utils_test.go → internal/repository/raw_test.go
View File

@ -1,4 +1,4 @@
package backend_test
package repository_test
import (
"bytes"
@ -10,6 +10,8 @@ import (
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/mem"
"github.com/restic/restic/internal/backend/mock"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
@ -19,9 +21,10 @@ const MiB = 1 << 20
func TestLoadAll(t *testing.T) {
b := mem.New()
var buf []byte
repo, err := repository.New(b, repository.Options{})
rtest.OK(t, err)
for i := 0; i < 20; i++ {
for i := 0; i < 5; i++ {
data := rtest.Random(23+i, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
@ -29,7 +32,7 @@ func TestLoadAll(t *testing.T) {
err := b.Save(context.TODO(), h, backend.NewByteReader(data, b.Hasher()))
rtest.OK(t, err)
buf, err := backend.LoadAll(context.TODO(), buf, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
buf, err := repo.LoadRaw(context.TODO(), backend.PackFile, id)
rtest.OK(t, err)
if len(buf) != len(data) {
@ -44,16 +47,6 @@ func TestLoadAll(t *testing.T) {
}
}
func save(t testing.TB, be backend.Backend, buf []byte) backend.Handle {
id := restic.Hash(buf)
h := backend.Handle{Name: id.String(), Type: backend.PackFile}
err := be.Save(context.TODO(), h, backend.NewByteReader(buf, be.Hasher()))
if err != nil {
t.Fatal(err)
}
return h
}
type quickRetryBackend struct {
backend.Backend
}
@ -69,6 +62,8 @@ func (be *quickRetryBackend) Load(ctx context.Context, h backend.Handle, length
func TestLoadAllBroken(t *testing.T) {
b := mock.NewBackend()
repo, err := repository.New(b, repository.Options{})
rtest.OK(t, err)
data := rtest.Random(23, rand.Intn(MiB)+500*KiB)
id := restic.Hash(data)
@ -80,70 +75,17 @@ func TestLoadAllBroken(t *testing.T) {
}
// must fail on first try
_, err := backend.LoadAll(context.TODO(), nil, b, backend.Handle{Type: backend.PackFile, Name: id.String()})
if err == nil {
t.Fatalf("missing expected error")
}
_, err = repo.LoadRaw(context.TODO(), backend.PackFile, id)
rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "missing expected ErrInvalidData error, got %v", err)
// must return the broken data after a retry
be := &quickRetryBackend{Backend: b}
buf, err := backend.LoadAll(context.TODO(), nil, be, backend.Handle{Type: backend.PackFile, Name: id.String()})
repo, err = repository.New(be, repository.Options{})
rtest.OK(t, err)
buf, err := repo.LoadRaw(context.TODO(), backend.PackFile, id)
rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "missing expected ErrInvalidData error, got %v", err)
if !bytes.Equal(buf, data) {
t.Fatalf("wrong data returned")
}
}
func TestLoadAllAppend(t *testing.T) {
b := mem.New()
h1 := save(t, b, []byte("foobar test string"))
randomData := rtest.Random(23, rand.Intn(MiB)+500*KiB)
h2 := save(t, b, randomData)
var tests = []struct {
handle backend.Handle
buf []byte
want []byte
}{
{
handle: h1,
buf: nil,
want: []byte("foobar test string"),
},
{
handle: h1,
buf: []byte("xxx"),
want: []byte("foobar test string"),
},
{
handle: h2,
buf: nil,
want: randomData,
},
{
handle: h2,
buf: make([]byte, 0, 200),
want: randomData,
},
{
handle: h2,
buf: []byte("foobarbaz"),
want: randomData,
},
}
for _, test := range tests {
t.Run("", func(t *testing.T) {
buf, err := backend.LoadAll(context.TODO(), test.buf, b, test.handle)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(buf, test.want) {
t.Errorf("wrong data returned, want %q, got %q", test.want, buf)
}
})
}
}

3
internal/repository/repair_pack_test.go
View File

@ -7,6 +7,7 @@ import (
"time"
"github.com/restic/restic/internal/backend"
backendtest "github.com/restic/restic/internal/backend/test"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -24,7 +25,7 @@ func listBlobs(repo restic.Repository) restic.BlobSet {
}
func replaceFile(t *testing.T, repo restic.Repository, h backend.Handle, damage func([]byte) []byte) {
buf, err := backend.LoadAll(context.TODO(), nil, repo.Backend(), h)
buf, err := backendtest.LoadAll(context.TODO(), repo.Backend(), h)
test.OK(t, err)
buf = damage(buf)
test.OK(t, repo.Backend().Remove(context.TODO(), h))

4
internal/repository/repository_test.go
View File

@ -9,13 +9,13 @@ import (
"math/rand"
"os"
"path/filepath"
"strings"
"testing"
"time"
"github.com/restic/restic/internal/backend"
"github.com/restic/restic/internal/backend/local"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/index"
"github.com/restic/restic/internal/repository"
"github.com/restic/restic/internal/restic"
@ -259,7 +259,7 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
if err == nil {
t.Fatal("missing expected error")
}
rtest.Assert(t, strings.Contains(err.Error(), "invalid data returned"), "unexpected error: %v", err)
rtest.Assert(t, errors.Is(err, restic.ErrInvalidData), "unexpected error: %v", err)
}
type damageOnceBackend struct {

5
internal/restic/repository.go
View File

@ -57,6 +57,11 @@ type Repository interface {
// LoadUnpacked loads and decrypts the file with the given type and ID.
LoadUnpacked(ctx context.Context, t FileType, id ID) (data []byte, err error)
SaveUnpacked(context.Context, FileType, []byte) (ID, error)
// LoadRaw reads all data stored in the backend for the file with id and filetype t.
// If the backend returns data that does not match the id, then the buffer is returned
// along with an error that is a restic.ErrInvalidData error.
LoadRaw(ctx context.Context, t FileType, id ID) (data []byte, err error)
}
type FileType = backend.FileType