mirror of https://github.com/borgbackup/borg.git
compress extension: build lz4 like zstd, see #3415
(cherry picked from commit 73a70082c2
)
This commit is contained in:
parent
49f76a1e32
commit
7d8ec62b50
36
setup.py
36
setup.py
|
@ -12,8 +12,12 @@ from distutils.core import Command
|
||||||
|
|
||||||
import textwrap
|
import textwrap
|
||||||
|
|
||||||
|
import setup_lz4
|
||||||
import setup_zstd
|
import setup_zstd
|
||||||
|
|
||||||
|
# True: use the shared liblz4 (>= TBD) from the system, False: use the bundled lz4 code
|
||||||
|
prefer_system_liblz4 = True
|
||||||
|
|
||||||
# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
|
# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
|
||||||
prefer_system_libzstd = True
|
prefer_system_libzstd = True
|
||||||
|
|
||||||
|
@ -142,15 +146,6 @@ def detect_openssl(prefixes):
|
||||||
return prefix
|
return prefix
|
||||||
|
|
||||||
|
|
||||||
def detect_lz4(prefixes):
|
|
||||||
for prefix in prefixes:
|
|
||||||
filename = os.path.join(prefix, 'include', 'lz4.h')
|
|
||||||
if os.path.exists(filename):
|
|
||||||
with open(filename, 'r') as fd:
|
|
||||||
if 'LZ4_decompress_safe' in fd.read():
|
|
||||||
return prefix
|
|
||||||
|
|
||||||
|
|
||||||
def detect_libb2(prefixes):
|
def detect_libb2(prefixes):
|
||||||
for prefix in prefixes:
|
for prefix in prefixes:
|
||||||
filename = os.path.join(prefix, 'include', 'blake2.h')
|
filename = os.path.join(prefix, 'include', 'blake2.h')
|
||||||
|
@ -164,7 +159,6 @@ include_dirs = []
|
||||||
library_dirs = []
|
library_dirs = []
|
||||||
define_macros = []
|
define_macros = []
|
||||||
crypto_libraries = ['crypto']
|
crypto_libraries = ['crypto']
|
||||||
compression_libraries = ['lz4']
|
|
||||||
|
|
||||||
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
|
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
|
||||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||||
|
@ -177,16 +171,17 @@ include_dirs.append(os.path.join(ssl_prefix, 'include'))
|
||||||
library_dirs.append(os.path.join(ssl_prefix, 'lib'))
|
library_dirs.append(os.path.join(ssl_prefix, 'lib'))
|
||||||
|
|
||||||
|
|
||||||
possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
|
possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
|
||||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||||
if os.environ.get('BORG_LZ4_PREFIX'):
|
if os.environ.get('BORG_LZ4_PREFIX'):
|
||||||
possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
|
possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX'))
|
||||||
lz4_prefix = detect_lz4(possible_lz4_prefixes)
|
liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
|
||||||
if lz4_prefix:
|
if prefer_system_liblz4 and liblz4_prefix:
|
||||||
include_dirs.append(os.path.join(lz4_prefix, 'include'))
|
print('Detected and preferring liblz4 over bundled LZ4')
|
||||||
library_dirs.append(os.path.join(lz4_prefix, 'lib'))
|
define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
|
||||||
elif not on_rtd:
|
liblz4_system = True
|
||||||
raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes)))
|
else:
|
||||||
|
liblz4_system = False
|
||||||
|
|
||||||
possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
|
possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
|
||||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||||
|
@ -772,7 +767,10 @@ cmdclass = {
|
||||||
ext_modules = []
|
ext_modules = []
|
||||||
if not on_rtd:
|
if not on_rtd:
|
||||||
compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
|
compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
|
||||||
libraries=compression_libraries, define_macros=define_macros)
|
define_macros=define_macros)
|
||||||
|
compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
|
||||||
|
system_prefix=liblz4_prefix, system=liblz4_system,
|
||||||
|
**compress_ext_kwargs)
|
||||||
compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
|
compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
|
||||||
system_prefix=libzstd_prefix, system=libzstd_system,
|
system_prefix=libzstd_prefix, system=libzstd_system,
|
||||||
multithreaded=False, legacy=False, **compress_ext_kwargs)
|
multithreaded=False, legacy=False, **compress_ext_kwargs)
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
# Support code for building a C extension with lz4 files
|
||||||
|
#
|
||||||
|
# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
|
||||||
|
# 2017-present, Thomas Waldmann (mods to make it more generic, code for lz4)
|
||||||
|
# All rights reserved.
|
||||||
|
#
|
||||||
|
# This software may be modified and distributed under the terms
|
||||||
|
# of the BSD license. See the LICENSE file for details.
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
# lz4 files, structure as seen in lz4 project repository:
|
||||||
|
|
||||||
|
lz4_sources = [
|
||||||
|
'lib/lz4.c',
|
||||||
|
]
|
||||||
|
|
||||||
|
lz4_includes = [
|
||||||
|
'lib',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def lz4_system_prefix(prefixes):
|
||||||
|
for prefix in prefixes:
|
||||||
|
filename = os.path.join(prefix, 'include', 'lz4.h')
|
||||||
|
if os.path.exists(filename):
|
||||||
|
with open(filename, 'r') as fd:
|
||||||
|
if 'LZ4_decompress_safe' in fd.read():
|
||||||
|
return prefix
|
||||||
|
|
||||||
|
|
||||||
|
def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
|
||||||
|
"""amend kwargs with lz4 stuff for a distutils.extension.Extension initialization.
|
||||||
|
|
||||||
|
bundled_path: relative (to this file) path to the bundled library source code files
|
||||||
|
system_prefix: where the system-installed library can be found
|
||||||
|
system: True: use the system-installed shared library, False: use the bundled library code
|
||||||
|
kwargs: distutils.extension.Extension kwargs that should be amended
|
||||||
|
returns: amended kwargs
|
||||||
|
"""
|
||||||
|
def multi_join(paths, *path_segments):
|
||||||
|
"""apply os.path.join on a list of paths"""
|
||||||
|
return [os.path.join(*(path_segments + (path, ))) for path in paths]
|
||||||
|
|
||||||
|
use_system = system and system_prefix is not None
|
||||||
|
|
||||||
|
sources = kwargs.get('sources', [])
|
||||||
|
if not use_system:
|
||||||
|
sources += multi_join(lz4_sources, bundled_path)
|
||||||
|
|
||||||
|
include_dirs = kwargs.get('include_dirs', [])
|
||||||
|
if use_system:
|
||||||
|
include_dirs += multi_join(['include'], system_prefix)
|
||||||
|
else:
|
||||||
|
include_dirs += multi_join(lz4_includes, bundled_path)
|
||||||
|
|
||||||
|
library_dirs = kwargs.get('library_dirs', [])
|
||||||
|
if use_system:
|
||||||
|
library_dirs += multi_join(['lib'], system_prefix)
|
||||||
|
|
||||||
|
libraries = kwargs.get('libraries', [])
|
||||||
|
if use_system:
|
||||||
|
libraries += ['lz4', ]
|
||||||
|
|
||||||
|
extra_compile_args = kwargs.get('extra_compile_args', [])
|
||||||
|
if not use_system:
|
||||||
|
extra_compile_args += [] # not used yet
|
||||||
|
|
||||||
|
ret = dict(**kwargs)
|
||||||
|
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
|
||||||
|
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
|
||||||
|
return ret
|
|
@ -0,0 +1,5 @@
|
||||||
|
#ifdef BORG_USE_LIBLZ4
|
||||||
|
#include <lz4.h>
|
||||||
|
#else
|
||||||
|
#include "lz4/lib/lz4.h"
|
||||||
|
#endif
|
|
@ -25,9 +25,9 @@ except ImportError:
|
||||||
|
|
||||||
from .helpers import Buffer, DecompressionError
|
from .helpers import Buffer, DecompressionError
|
||||||
|
|
||||||
API_VERSION = '1.1_04'
|
API_VERSION = '1.1_05'
|
||||||
|
|
||||||
cdef extern from "lz4.h":
|
cdef extern from "algorithms/lz4-libselect.h":
|
||||||
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
||||||
int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
||||||
int LZ4_compressBound(int inputSize) nogil
|
int LZ4_compressBound(int inputSize) nogil
|
||||||
|
|
|
@ -135,7 +135,7 @@ def check_extension_modules():
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
if chunker.API_VERSION != '1.1_01':
|
if chunker.API_VERSION != '1.1_01':
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
if compress.API_VERSION != '1.1_04':
|
if compress.API_VERSION != '1.1_05':
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
if borg.crypto.low_level.API_VERSION != '1.1_02':
|
if borg.crypto.low_level.API_VERSION != '1.1_02':
|
||||||
raise ExtensionModuleError
|
raise ExtensionModuleError
|
||||||
|
|
Loading…
Reference in New Issue