mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #1472 from ThomasWaldmann/lz4-buffer 

lz4 reusing helpers.Buffer
This commit is contained in:
TW 2016-08-13 02:13:11 +02:00 committed by GitHub
parent df967722e3 e1bc7b62f6
commit feb7e2517e
1 changed files with 7 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
borg/compress.pyx
View File

@ -1,25 +1,18 @@
import threading
import zlib import zlib
try: try:
import lzma import lzma
except ImportError: except ImportError:
lzma = None lzma = None
from .helpers import Buffer
cdef extern from "lz4.h": cdef extern from "lz4.h":
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
int LZ4_compressBound(int inputSize) nogil int LZ4_compressBound(int inputSize) nogil
thread_local = threading.local() buffer = Buffer(bytearray, size=0)
thread_local.buffer = bytes()
cdef char *get_buffer(size):
size = int(size)
if len(thread_local.buffer) < size:
thread_local.buffer = bytes(size)
return <char *> thread_local.buffer
cdef class CompressorBase: cdef class CompressorBase:
@ -88,7 +81,8 @@ class LZ4(CompressorBase):
cdef char *source = idata cdef char *source = idata
cdef char *dest cdef char *dest
osize = LZ4_compressBound(isize) osize = LZ4_compressBound(isize)
dest = get_buffer(osize) buf = buffer.get(osize)
dest = <char *> buf
with nogil: with nogil:
osize = LZ4_compress_limitedOutput(source, dest, isize, osize) osize = LZ4_compress_limitedOutput(source, dest, isize, osize)
if not osize: if not osize:
@ -108,7 +102,8 @@ class LZ4(CompressorBase):
# allocate more if isize * 3 is already bigger, to avoid having to resize often. # allocate more if isize * 3 is already bigger, to avoid having to resize often.
osize = max(int(1.1 * 2**23), isize * 3) osize = max(int(1.1 * 2**23), isize * 3)
while True: while True:
dest = get_buffer(osize) buf = buffer.get(osize)
dest = <char *> buf
with nogil: with nogil:
rsize = LZ4_decompress_safe(source, dest, isize, osize) rsize = LZ4_decompress_safe(source, dest, isize, osize)
if rsize >= 0: if rsize >= 0: