mirror of
https://github.com/restic/restic.git
synced 2024-12-23 00:07:25 +00:00
Add chunker pool
This commit is contained in:
parent
3daeee8e2c
commit
bbb3145f11
2 changed files with 54 additions and 11 deletions
|
@ -1,7 +1,6 @@
|
||||||
package restic
|
package restic
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
@ -18,6 +17,8 @@ import (
|
||||||
const (
|
const (
|
||||||
maxConcurrentFiles = 16
|
maxConcurrentFiles = 16
|
||||||
maxConcurrentBlobs = 16
|
maxConcurrentBlobs = 16
|
||||||
|
|
||||||
|
// chunkerBufSize is used in pool.go
|
||||||
chunkerBufSize = 512 * chunker.KiB
|
chunkerBufSize = 512 * chunker.KiB
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -161,9 +162,10 @@ func (arch *Archiver) SaveFile(node *Node) (Blobs, error) {
|
||||||
var blobs Blobs
|
var blobs Blobs
|
||||||
|
|
||||||
// store all chunks
|
// store all chunks
|
||||||
chnker := chunker.New(file, chunkerBufSize, sha256.New)
|
chnker := GetChunker("archiver.SaveFile")
|
||||||
|
chnker.Reset(file)
|
||||||
chans := [](<-chan Blob){}
|
chans := [](<-chan Blob){}
|
||||||
//defer chnker.Free()
|
defer FreeChunker("archiver.SaveFile", chnker)
|
||||||
|
|
||||||
chunks := 0
|
chunks := 0
|
||||||
|
|
||||||
|
|
47
pools.go
47
pools.go
|
@ -1,8 +1,10 @@
|
||||||
package restic
|
package restic
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"github.com/restic/restic/chunker"
|
||||||
"github.com/restic/restic/debug"
|
"github.com/restic/restic/debug"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -12,6 +14,7 @@ type poolStats struct {
|
||||||
mput map[string]int
|
mput map[string]int
|
||||||
mmax map[string]int
|
mmax map[string]int
|
||||||
|
|
||||||
|
new int
|
||||||
get int
|
get int
|
||||||
put int
|
put int
|
||||||
max int
|
max int
|
||||||
|
@ -65,21 +68,40 @@ func newPoolStats() *poolStats {
|
||||||
var (
|
var (
|
||||||
chunkPool = sync.Pool{New: newChunkBuf}
|
chunkPool = sync.Pool{New: newChunkBuf}
|
||||||
nodePool = sync.Pool{New: newNode}
|
nodePool = sync.Pool{New: newNode}
|
||||||
|
chunkerPool = sync.Pool{New: newChunker}
|
||||||
|
|
||||||
chunkStats = newPoolStats()
|
chunkStats = newPoolStats()
|
||||||
nodeStats = newPoolStats()
|
nodeStats = newPoolStats()
|
||||||
|
chunkerStats = newPoolStats()
|
||||||
)
|
)
|
||||||
|
|
||||||
func newChunkBuf() interface{} {
|
func newChunkBuf() interface{} {
|
||||||
|
chunkStats.m.Lock()
|
||||||
|
defer chunkStats.m.Unlock()
|
||||||
|
chunkStats.new++
|
||||||
|
|
||||||
// create buffer for iv, data and hmac
|
// create buffer for iv, data and hmac
|
||||||
return make([]byte, maxCiphertextSize)
|
return make([]byte, maxCiphertextSize)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newNode() interface{} {
|
func newNode() interface{} {
|
||||||
|
nodeStats.m.Lock()
|
||||||
|
defer nodeStats.m.Unlock()
|
||||||
|
nodeStats.new++
|
||||||
|
|
||||||
// create buffer for iv, data and hmac
|
// create buffer for iv, data and hmac
|
||||||
return new(Node)
|
return new(Node)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newChunker() interface{} {
|
||||||
|
chunkStats.m.Lock()
|
||||||
|
defer chunkStats.m.Unlock()
|
||||||
|
chunkStats.new++
|
||||||
|
|
||||||
|
// create a new chunker with a nil reader
|
||||||
|
return chunker.New(nil, chunkerBufSize, sha256.New)
|
||||||
|
}
|
||||||
|
|
||||||
func GetChunkBuf(s string) []byte {
|
func GetChunkBuf(s string) []byte {
|
||||||
chunkStats.Get(s)
|
chunkStats.Get(s)
|
||||||
return chunkPool.Get().([]byte)
|
return chunkPool.Get().([]byte)
|
||||||
|
@ -100,14 +122,33 @@ func FreeNode(n *Node) {
|
||||||
nodePool.Put(n)
|
nodePool.Put(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GetChunker(s string) *chunker.Chunker {
|
||||||
|
chunkerStats.Get(s)
|
||||||
|
return chunkerPool.Get().(*chunker.Chunker)
|
||||||
|
}
|
||||||
|
|
||||||
|
func FreeChunker(s string, ch *chunker.Chunker) {
|
||||||
|
chunkerStats.Put(s)
|
||||||
|
chunkerPool.Put(ch)
|
||||||
|
}
|
||||||
|
|
||||||
func PoolAlloc() {
|
func PoolAlloc() {
|
||||||
debug.Log("pools.PoolAlloc", "pool stats for chunk: get %d, put %d, diff %d, max %d\n", chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
|
debug.Log("pools.PoolAlloc", "pool stats for chunk: new %d, get %d, put %d, diff %d, max %d\n",
|
||||||
|
chunkStats.new, chunkStats.get, chunkStats.put, chunkStats.get-chunkStats.put, chunkStats.max)
|
||||||
for k, v := range chunkStats.mget {
|
for k, v := range chunkStats.mget {
|
||||||
debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
|
debug.Log("pools.PoolAlloc", "pool stats for chunk[%s]: get %d, put %d, diff %d, max %d\n",
|
||||||
|
k, v, chunkStats.mput[k], v-chunkStats.mput[k], chunkStats.mmax[k])
|
||||||
}
|
}
|
||||||
|
|
||||||
debug.Log("pools.PoolAlloc", "pool stats for node: get %d, put %d, diff %d, max %d\n", nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
|
debug.Log("pools.PoolAlloc", "pool stats for node: new %d, get %d, put %d, diff %d, max %d\n",
|
||||||
|
nodeStats.new, nodeStats.get, nodeStats.put, nodeStats.get-nodeStats.put, nodeStats.max)
|
||||||
for k, v := range nodeStats.mget {
|
for k, v := range nodeStats.mget {
|
||||||
debug.Log("pools.PoolAlloc", "pool stats for node[%s]: get %d, put %d, diff %d, max %d\n", k, v, nodeStats.mput[k], v-nodeStats.mput[k], nodeStats.mmax[k])
|
debug.Log("pools.PoolAlloc", "pool stats for node[%s]: get %d, put %d, diff %d, max %d\n", k, v, nodeStats.mput[k], v-nodeStats.mput[k], nodeStats.mmax[k])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
debug.Log("pools.PoolAlloc", "pool stats for chunker: new %d, get %d, put %d, diff %d, max %d\n",
|
||||||
|
chunkerStats.new, chunkerStats.get, chunkerStats.put, chunkerStats.get-chunkerStats.put, chunkerStats.max)
|
||||||
|
for k, v := range chunkerStats.mget {
|
||||||
|
debug.Log("pools.PoolAlloc", "pool stats for chunker[%s]: get %d, put %d, diff %d, max %d\n", k, v, chunkerStats.mput[k], v-chunkerStats.mput[k], chunkerStats.mmax[k])
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue