From ea22b2dfb1e55c8cd22e4983d147cb83c07cee41 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 5 May 2015 00:45:29 +0200 Subject: [PATCH] chunker: move buffer pool to chunker package --- archiver.go | 10 +--------- archiver_test.go | 14 ++++++-------- chunker/chunker.go | 21 ++++++++++++--------- chunker/chunker_test.go | 23 +++++++---------------- 4 files changed, 26 insertions(+), 42 deletions(-) diff --git a/archiver.go b/archiver.go index e994502ef..cc5d079b1 100644 --- a/archiver.go +++ b/archiver.go @@ -170,12 +170,6 @@ func updateNodeContent(node *Node, results []saveResult) error { return nil } -const chunkerBufSize = 512 * chunker.KiB - -var chunkerBufPool = sync.Pool{ - New: func() interface{} { return make([]byte, chunkerBufSize) }, -} - // SaveFile stores the content of the file on the backend as a Blob by calling // Save for each chunk. func (arch *Archiver) SaveFile(p *Progress, node *Node) error { @@ -190,9 +184,7 @@ func (arch *Archiver) SaveFile(p *Progress, node *Node) error { return err } - buf := chunkerBufPool.Get().([]byte) - chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, buf, sha256.New()) - defer chunkerBufPool.Put(buf) + chnker := chunker.New(file, arch.s.Config.ChunkerPolynomial, sha256.New()) resultChannels := [](<-chan saveResult){} for { diff --git a/archiver_test.go b/archiver_test.go index 8babeed14..331da66ce 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -25,9 +25,9 @@ type Rdr interface { io.ReaderAt } -func benchmarkChunkEncrypt(b testing.TB, buf, buf2, chunkBuf []byte, rd Rdr, key *crypto.Key) { +func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.Key) { rd.Seek(0, 0) - ch := chunker.New(rd, testPol, chunkBuf, sha256.New()) + ch := chunker.New(rd, testPol, sha256.New()) for { chunk, err := ch.Next() @@ -58,18 +58,17 @@ func BenchmarkChunkEncrypt(b *testing.B) { buf := make([]byte, chunker.MaxSize) buf2 := make([]byte, chunker.MaxSize) - chunkBuf := make([]byte, chunkerBufSize) b.ResetTimer() b.SetBytes(int64(len(data))) for i := 0; i < b.N; i++ { - benchmarkChunkEncrypt(b, buf, buf2, chunkBuf, rd, s.Key()) + benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key()) } } -func benchmarkChunkEncryptP(b *testing.PB, buf, chunkBuf []byte, rd Rdr, key *crypto.Key) { - ch := chunker.New(rd, testPol, chunkBuf, sha256.New()) +func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) { + ch := chunker.New(rd, testPol, sha256.New()) for { chunk, err := ch.Next() @@ -91,7 +90,6 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { data := Random(23, 10<<20) // 10MiB buf := make([]byte, chunker.MaxSize) - chunkBuf := make([]byte, chunkerBufSize) b.ResetTimer() b.SetBytes(int64(len(data))) @@ -99,7 +97,7 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { rd := bytes.NewReader(data) - benchmarkChunkEncryptP(pb, buf, chunkBuf, rd, s.Key()) + benchmarkChunkEncryptP(pb, buf, rd, s.Key()) } }) } diff --git a/chunker/chunker.go b/chunker/chunker.go index 11d85b811..a7a5609b7 100644 --- a/chunker/chunker.go +++ b/chunker/chunker.go @@ -23,8 +23,14 @@ const ( MaxSize = 8 * MiB splitmask = (1 << averageBits) - 1 + + chunkerBufSize = 512 * KiB ) +var bufPool = sync.Pool{ + New: func() interface{} { return make([]byte, chunkerBufSize) }, +} + type tables struct { out [256]Pol mod [256]Pol @@ -79,18 +85,12 @@ type Chunker struct { h hash.Hash } -const minBufSize = 32 - // New returns a new Chunker based on polynomial p that reads from rd // with bufsize and pass all data to hash along the way, using buf for -// buffering. Buf must at least hold 32 bytes. -func New(rd io.Reader, pol Pol, buf []byte, h hash.Hash) *Chunker { - if len(buf) < minBufSize { - buf = make([]byte, minBufSize) - } - +// buffering. +func New(rd io.Reader, pol Pol, h hash.Hash) *Chunker { c := &Chunker{ - buf: buf, + buf: bufPool.Get().([]byte), h: h, pol: pol, rd: rd, @@ -204,6 +204,9 @@ func (c *Chunker) Next() (*Chunk, error) { if err == io.EOF && !c.closed { c.closed = true + // return the buffer to the pool + bufPool.Put(c.buf) + // return current chunk, if any bytes have been processed if c.count > 0 { return &Chunk{ diff --git a/chunker/chunker_test.go b/chunker/chunker_test.go index 7799a6db1..5c38d2b0d 100644 --- a/chunker/chunker_test.go +++ b/chunker/chunker_test.go @@ -147,13 +147,10 @@ func getRandom(seed, count int) []byte { return buf } -const chunkerBufSize = 512 * chunker.KiB - func TestChunker(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) - chunkBuf := make([]byte, chunkerBufSize) - ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) + ch := chunker.New(bytes.NewReader(buf), testPol, sha256.New()) chunks := testWithData(t, ch, chunks1) // test reader @@ -180,7 +177,7 @@ func TestChunker(t *testing.T) { // setup nullbyte data source buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize) - ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) + ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New()) testWithData(t, ch, chunks2) } @@ -188,7 +185,6 @@ func TestChunker(t *testing.T) { func TestChunkerWithRandomPolynomial(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) - chunkBuf := make([]byte, chunkerBufSize) // generate a new random polynomial start := time.Now() @@ -197,7 +193,7 @@ func TestChunkerWithRandomPolynomial(t *testing.T) { t.Logf("generating random polynomial took %v", time.Since(start)) start = time.Now() - ch := chunker.New(bytes.NewReader(buf), p, chunkBuf, sha256.New()) + ch := chunker.New(bytes.NewReader(buf), p, sha256.New()) t.Logf("creating chunker took %v", time.Since(start)) // make sure that first chunk is different @@ -214,9 +210,8 @@ func TestChunkerWithRandomPolynomial(t *testing.T) { func TestChunkerWithoutHash(t *testing.T) { // setup data source buf := getRandom(23, 32*1024*1024) - chunkBuf := make([]byte, chunkerBufSize) - ch := chunker.New(bytes.NewReader(buf), testPol, chunkBuf, nil) + ch := chunker.New(bytes.NewReader(buf), testPol, nil) chunks := testWithData(t, ch, chunks1) // test reader @@ -246,7 +241,7 @@ func TestChunkerWithoutHash(t *testing.T) { // setup nullbyte data source buf = bytes.Repeat([]byte{0}, len(chunks2)*chunker.MinSize) - ch = chunker.New(bytes.NewReader(buf), testPol, chunkBuf, sha256.New()) + ch = chunker.New(bytes.NewReader(buf), testPol, sha256.New()) testWithData(t, ch, chunks2) } @@ -276,8 +271,6 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) { rd = bytes.NewReader(getRandom(23, size)) } - chunkBuf := make([]byte, chunkerBufSize) - b.ResetTimer() b.SetBytes(int64(size)) @@ -286,7 +279,7 @@ func benchmarkChunker(b *testing.B, hash hash.Hash) { chunks = 0 rd.Seek(0, 0) - ch := chunker.New(rd, testPol, chunkBuf, hash) + ch := chunker.New(rd, testPol, hash) for { _, err := ch.Next() @@ -322,11 +315,9 @@ func BenchmarkNewChunker(b *testing.B) { p, err := chunker.RandomPolynomial() OK(b, err) - chunkBuf := make([]byte, chunkerBufSize) - b.ResetTimer() for i := 0; i < b.N; i++ { - chunker.New(bytes.NewBuffer(nil), p, chunkBuf, nil) + chunker.New(bytes.NewBuffer(nil), p, nil) } }