mirror/restic
1
0
Fork 0
mirror of https://github.com/restic/restic.git synced 2025-01-03 13:45:20 +00:00
Code Issues Releases Wiki Activity

checker: reuse bufio reader

Browse source
This commit is contained in:
Michael Eischer 2021-08-22 14:38:27 +02:00
parent 34ebafb8b6
commit 930a00ad54
2 changed files with 7 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
internal/checker/checker.go
View file

@ -440,7 +440,7 @@ func (c *Checker) GetPacks() map[restic.ID]int64 {
} }
// checkPack reads a pack and checks the integrity of all blobs. // checkPack reads a pack and checks the integrity of all blobs.
func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64) error { func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader) error {
debug.Log("checking pack %v", id.String()) debug.Log("checking pack %v", id.String())
if len(blobs) == 0 { if len(blobs) == 0 {
@ -474,14 +474,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r
hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { hashingLoader := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error {
hrd := hashing.NewReader(rd, sha256.New()) hrd := hashing.NewReader(rd, sha256.New())
bufRd.Reset(hrd)
// create a buffer that is large enough to be reused by repository.StreamPack
// this ensures that we can read the pack header later on
bufferSize := int(size)
if bufferSize > repository.MaxStreamBufferSize {
bufferSize = repository.MaxStreamBufferSize
}
bufRd := bufio.NewReaderSize(hrd, bufferSize)
// skip to start of first blob, offset == 0 for correct pack files // skip to start of first blob, offset == 0 for correct pack files
_, err := bufRd.Discard(int(offset)) _, err := bufRd.Discard(int(offset))
@ -587,6 +580,9 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
// run workers // run workers
for i := 0; i < defaultParallelism; i++ { for i := 0; i < defaultParallelism; i++ {
g.Go(func() error { g.Go(func() error {
// create a buffer that is large enough to be reused by repository.StreamPack
// this ensures that we can read the pack header later on
bufRd := bufio.NewReaderSize(nil, repository.MaxStreamBufferSize)
for { for {
var ps checkTask var ps checkTask
var ok bool var ok bool
@ -600,7 +596,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p
} }
} }
err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size) err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd)
p.Add(1) p.Add(1)
if err == nil { if err == nil {
continue continue

1
internal/repository/repository.go
View file

@ -767,6 +767,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack
if bufferSize > MaxStreamBufferSize { if bufferSize > MaxStreamBufferSize {
bufferSize = MaxStreamBufferSize bufferSize = MaxStreamBufferSize
} }
// create reader here to allow reusing the buffered reader from checker.checkData
bufRd := bufio.NewReaderSize(rd, bufferSize) bufRd := bufio.NewReaderSize(rd, bufferSize)
currentBlobEnd := dataStart currentBlobEnd := dataStart
var buf []byte var buf []byte