mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 09:19:31 +00:00
Code Issues Releases Wiki Activity

fixup: use thread-local buffer

Browse source 
start with 0 bytes length (saves memory in case lz4 is not used).
always grow when a bigger buffer is needed.
avoid per-call reallocation / freeing / garbage.
This commit is contained in:
Thomas Waldmann 2016-08-09 17:05:24 +02:00
parent d3000a7e5d
commit b0e7bb5ddc
1 changed files with 15 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
borg/compress.pyx
View file

@ -1,3 +1,4 @@
import threading
import zlib
try:
import lzma
@ -10,6 +11,17 @@ cdef extern from "lz4.h":
int LZ4_compressBound(int inputSize) nogil
thread_local = threading.local()
thread_local.buffer = bytes()
cdef char *get_buffer(size):
size = int(size)
if len(thread_local.buffer) < size:
thread_local.buffer = bytes(size)
return <char *> thread_local.buffer
cdef class CompressorBase:
"""
base class for all (de)compression classes,
@ -66,11 +78,7 @@ class LZ4(CompressorBase):
name = 'lz4'
def __init__(self, **kwargs):
self.buffer = None
def _create_buffer(self, size):
# we keep a reference to the buffer until this instance is destroyed
self.buffer = bytes(int(size))
pass
def compress(self, idata):
if not isinstance(idata, bytes):
@ -80,8 +88,7 @@ class LZ4(CompressorBase):
cdef char *source = idata
cdef char *dest
osize = LZ4_compressBound(isize)
self._create_buffer(osize)
dest = self.buffer
dest = get_buffer(osize)
with nogil:
osize = LZ4_compress_limitedOutput(source, dest, isize, osize)
if not osize:
@ -101,8 +108,7 @@ class LZ4(CompressorBase):
# allocate more if isize * 3 is already bigger, to avoid having to resize often.
osize = max(int(1.1 * 2**23), isize * 3)
while True:
self._create_buffer(osize)
dest = self.buffer
dest = get_buffer(osize)
with nogil:
rsize = LZ4_decompress_safe(source, dest, isize, osize)
if rsize >= 0: