diff --git a/archiver.go b/archiver.go index cc5d079b1..e994502ef 100644 --- a/archiver.go +++ b/archiver.go @@ -170,6 +170,12 @@ func updateNodeContent(node *Node, results []saveResult) error { return nil } +const chunkerBufSize = 512 * chunker.KiB + +var chunkerBufPool = sync.Pool{ + New: func() interface{} { return make([]byte, chunkerBufSize) }, +} + // SaveFile stores the content of the file on the backend as a Blob by calling // Save for each chunk. func (arch *Archiver) SaveFile(p *Progress, node *Node) error { @@ -184,7 +190,9 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error { return err } - chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New()) + buf := chunkerBufPool.Get().([]byte) + chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New()) + defer chunkerBufPool.Put(buf) resultChannels := [](<-chan saveResult){} for { diff --git a/archiver_test.go b/archiver_test.go index 519f34839..d7ef839eb 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -18,16 +18,16 @@ import ( var benchArchiveDirectory = flag.String("test.benchdir", ".", "benchmark archiving a real directory (default: .)") var testPol = chunker.Pol(0x3DA3358B4DC173) -const bufSize = chunker.MiB +const chunkerBufSize = 512 * chunker.KiB type Rdr interface { io.ReadSeeker io.ReaderAt } -func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) { +func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) { rd.Seek(0, 0) - ch := chunker.New(rd, testPol, sha256.New()) + ch := chunker.New(rd, testPol, chunkBuf, sha256.New()) for { chunk, err := ch.Next() @@ -58,20 +58,21 @@ func BenchmarkChunkEncrypt(b *testing.B) { buf := restic.GetChunkBuf("BenchmarkChunkEncrypt") buf2 := restic.GetChunkBuf("BenchmarkChunkEncrypt") + chunkBuf := make([]byte, chunkerBufSize) b.ResetTimer() b.SetBytes(int64(len(data))) for i := 0; i < b.N; i++ { - benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key()) + benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key()) } restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf) restic.FreeChunkBuf("BenchmarkChunkEncrypt", buf2) } -func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) { - ch := chunker.New(rd, testPol, sha256.New()) +func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) { + ch := chunker.New(rd, testPol, chunkBuf, sha256.New()) for { chunk, err := ch.Next() @@ -93,6 +94,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { data := Random(23, 10<<20) // 10MiB buf := restic.GetChunkBuf("BenchmarkChunkEncryptParallel") + chunkBuf := make([]byte, chunkerBufSize) b.ResetTimer() b.SetBytes(int64(len(data))) @@ -100,7 +102,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { rd := bytes.NewReader(data) - benchmarkChunkEncryptP(pb, buf, rd, s.Key()) + benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key()) } }) diff --git a/chunker/chunker.go b/chunker/chunker.go index 944321f75..33f5b1504 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -82,11 +82,18 @@ type Chunker struct { h hash.Hash } -// New returns a new Chunker based on polynomial p that reads from data from rd -// with bufsize and pass all data to hash along the way. -func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker { +const minBufSize = 32 + +// New returns a new Chunker based on polynomial p that reads from rd +// with bufsize and pass all data to hash along the way, using buf for +// buffering. Buf must at least hold 32 bytes. +func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker { + if len(buf) < minBufSize { + buf = make([]byte, minBufSize) + } + c := &Chunker{ - buf: make([]byte, bufSize), + buf: buf, h: h, pol: pol, rd: rd, diff --git a/chunker/chunker_test.go b/chunker/chunker_test.go index fbd692d7d..7799a6db1 100644 --- a/chunker/chunker_test.go +++ b/chunker/chunker_test.go @@ -147,10 +147,13 @@ func getRandom(seed, count int) []byte { return buf } +const chunkerBufSize = 512 * chunker.KiB + func TestChunker(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) - ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New()) + chunkBuf := make([]byte, chunkerBufSize) + ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) chunks := testWithData(t, ch, chunks1) // test reader @@ -177,7 +180,7 @@ func TestChunker(t *testing.T) { // setup nullbyte data source buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize) - ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New()) + ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) testWithData(t, ch, chunks2) } @@ -185,6 +188,7 @@ func TestChunker(t *testing.T) { func TestChunkerWithRandomPolynomial(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) + chunkBuf := make([]byte, chunkerBufSize) // generate a new random polynomial start := time.Now() @@ -193,7 +197,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) { t.Logf("generating random polynomial took %v", time.Since(start)) start = time.Now() - ch := chunker.New(bytes.NewReader(buf), p, sha256.New()) + ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New()) t.Logf("creating chunker took %v", time.Since(start)) // make sure that first chunk is different @@ -210,7 +214,9 @@ func TestChunkerWithRandomPolynomial(t *testing.T) { func TestChunkerWithoutHash(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) - ch := chunker.New(bytes.NewReader(buf), testPol, nil) + chunkBuf := make([]byte, chunkerBufSize) + + ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil) chunks := testWithData(t, ch, chunks1) // test reader @@ -240,7 +246,7 @@ func TestChunkerWithoutHash(t *testing.T) { // setup nullbyte data source buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize) - ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New()) + ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) testWithData(t, ch, chunks2) } @@ -270,6 +276,8 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) { rd = bytes.NewReader(getRandom(23, size)) } + chunkBuf := make([]byte, chunkerBufSize) + b.ResetTimer() b.SetBytes(int64(size)) @@ -278,7 +286,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) { chunks = 0 rd.Seek(0, 0) - ch := chunker.New(rd, testPol, hash) + ch := chunker.New(rd, testPol, chunkBuf, hash) for { _, err := ch.Next() @@ -314,9 +322,11 @@ func BenchmarkNewChunker(b *testing.B) { p, err := chunker.RandomPolynomial() OK(b, err) + chunkBuf := make([]byte, chunkerBufSize) + b.ResetTimer() for i := 0; i < b.N; i++ { - chunker.New(bytes.NewBuffer(nil), p, nil) + chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil) } }