diff --git a/borg/compress.pyx b/borg/compress.pyx index 1330fbf2f..13955a86b 100644 --- a/borg/compress.pyx +++ b/borg/compress.pyx @@ -1,3 +1,4 @@ +import threading import zlib try: import lzma @@ -10,6 +11,17 @@ cdef extern from "lz4.h": int LZ4_compressBound(int inputSize) nogil +thread_local = threading.local() +thread_local.buffer = bytes() + + +cdef char *get_buffer(size): + size = int(size) + if len(thread_local.buffer) < size: + thread_local.buffer = bytes(size) + return thread_local.buffer + + cdef class CompressorBase: """ base class for all (de)compression classes, @@ -66,11 +78,7 @@ class LZ4(CompressorBase): name = 'lz4' def __init__(self, **kwargs): - self.buffer = None - - def _create_buffer(self, size): - # we keep a reference to the buffer until this instance is destroyed - self.buffer = bytes(int(size)) + pass def compress(self, idata): if not isinstance(idata, bytes): @@ -80,8 +88,7 @@ class LZ4(CompressorBase): cdef char *source = idata cdef char *dest osize = LZ4_compressBound(isize) - self._create_buffer(osize) - dest = self.buffer + dest = get_buffer(osize) with nogil: osize = LZ4_compress_limitedOutput(source, dest, isize, osize) if not osize: @@ -101,8 +108,7 @@ class LZ4(CompressorBase): # allocate more if isize * 3 is already bigger, to avoid having to resize often. osize = max(int(1.1 * 2**23), isize * 3) while True: - self._create_buffer(osize) - dest = self.buffer + dest = get_buffer(osize) with nogil: rsize = LZ4_decompress_safe(source, dest, isize, osize) if rsize >= 0: