1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2024-12-23 08:16:36 +00:00
restic/internal/archiver/blob_saver.go
Michael Eischer ce89018902 Fix data race in blob_saver
After the `BlobSaver` job is submitted, the buffer can be released and
reused by another `FileSaver` even before `BlobSaver.Save` returns. That
FileSaver will the change `buf.Data` leading to wrong backup statistics.

Found by `go test -race ./...`:

WARNING: DATA RACE
Write at 0x00c0000784a0 by goroutine 41:
  github.com/restic/restic/internal/archiver.(*FileSaver).saveFile()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:176 +0x789
  github.com/restic/restic/internal/archiver.(*FileSaver).worker()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af
  github.com/restic/restic/internal/archiver.NewFileSaver.func2()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d
  golang.org/x/sync/errgroup.(*Group).Go.func1()
      /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91

Previous read at 0x00c0000784a0 by goroutine 29:
  github.com/restic/restic/internal/archiver.(*BlobSaver).Save()
      /home/michael/Projekte/restic/restic/internal/archiver/blob_saver.go:57 +0x1dd
  github.com/restic/restic/internal/archiver.(*BlobSaver).Save-fm()
      <autogenerated>:1 +0xac
  github.com/restic/restic/internal/archiver.(*FileSaver).saveFile()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:191 +0x855
  github.com/restic/restic/internal/archiver.(*FileSaver).worker()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:242 +0x2af
  github.com/restic/restic/internal/archiver.NewFileSaver.func2()
      /home/michael/Projekte/restic/restic/internal/archiver/file_saver.go:88 +0x5d
  golang.org/x/sync/errgroup.(*Group).Go.func1()
      /home/michael/go/pkg/mod/golang.org/x/sync@v0.0.0-20210220032951-036812b2e83c/errgroup/errgroup.go:57 +0x91
2022-07-03 14:47:53 +02:00

151 lines
3.3 KiB
Go

package archiver
import (
"context"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
"golang.org/x/sync/errgroup"
)
// Saver allows saving a blob.
type Saver interface {
SaveBlob(ctx context.Context, t restic.BlobType, data []byte, id restic.ID, storeDuplicate bool) (restic.ID, bool, int, error)
Index() restic.MasterIndex
}
// BlobSaver concurrently saves incoming blobs to the repo.
type BlobSaver struct {
repo Saver
ch chan<- saveBlobJob
}
// NewBlobSaver returns a new blob. A worker pool is started, it is stopped
// when ctx is cancelled.
func NewBlobSaver(ctx context.Context, wg *errgroup.Group, repo Saver, workers uint) *BlobSaver {
ch := make(chan saveBlobJob)
s := &BlobSaver{
repo: repo,
ch: ch,
}
for i := uint(0); i < workers; i++ {
wg.Go(func() error {
return s.worker(ctx, ch)
})
}
return s
}
func (s *BlobSaver) TriggerShutdown() {
close(s.ch)
}
// Save stores a blob in the repo. It checks the index and the known blobs
// before saving anything. It takes ownership of the buffer passed in.
func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer) FutureBlob {
// buf might be freed once the job was submitted, thus calculate the length now
length := len(buf.Data)
ch := make(chan saveBlobResponse, 1)
select {
case s.ch <- saveBlobJob{BlobType: t, buf: buf, ch: ch}:
case <-ctx.Done():
debug.Log("not sending job, context is cancelled")
close(ch)
return FutureBlob{ch: ch}
}
return FutureBlob{ch: ch, length: length}
}
// FutureBlob is returned by SaveBlob and will return the data once it has been processed.
type FutureBlob struct {
ch <-chan saveBlobResponse
length int
res saveBlobResponse
}
// Wait blocks until the result is available or the context is cancelled.
func (s *FutureBlob) Wait(ctx context.Context) {
select {
case <-ctx.Done():
return
case res, ok := <-s.ch:
if ok {
s.res = res
}
}
}
// ID returns the ID of the blob after it has been saved.
func (s *FutureBlob) ID() restic.ID {
return s.res.id
}
// Known returns whether or not the blob was already known.
func (s *FutureBlob) Known() bool {
return s.res.known
}
// Length returns the raw length of the blob.
func (s *FutureBlob) Length() int {
return s.length
}
// SizeInRepo returns the number of bytes added to the repo (including
// compression and crypto overhead).
func (s *FutureBlob) SizeInRepo() int {
return s.res.size
}
type saveBlobJob struct {
restic.BlobType
buf *Buffer
ch chan<- saveBlobResponse
}
type saveBlobResponse struct {
id restic.ID
known bool
size int
}
func (s *BlobSaver) saveBlob(ctx context.Context, t restic.BlobType, buf []byte) (saveBlobResponse, error) {
id, known, size, err := s.repo.SaveBlob(ctx, t, buf, restic.ID{}, false)
if err != nil {
return saveBlobResponse{}, err
}
return saveBlobResponse{
id: id,
known: known,
size: size,
}, nil
}
func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error {
for {
var job saveBlobJob
var ok bool
select {
case <-ctx.Done():
return nil
case job, ok = <-jobs:
if !ok {
return nil
}
}
res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data)
if err != nil {
debug.Log("saveBlob returned error, exiting: %v", err)
close(job.ch)
return err
}
job.ch <- res
close(job.ch)
job.buf.Release()
}
}