diff --git a/setup.py b/setup.py index 036bfc106..bca822996 100644 --- a/setup.py +++ b/setup.py @@ -12,8 +12,12 @@ from distutils.core import Command import textwrap +import setup_lz4 import setup_zstd +# True: use the shared liblz4 (>= TBD) from the system, False: use the bundled lz4 code +prefer_system_liblz4 = True + # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code prefer_system_libzstd = True @@ -142,15 +146,6 @@ def detect_openssl(prefixes): return prefix -def detect_lz4(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'lz4.h') - if os.path.exists(filename): - with open(filename, 'r') as fd: - if 'LZ4_decompress_safe' in fd.read(): - return prefix - - def detect_libb2(prefixes): for prefix in prefixes: filename = os.path.join(prefix, 'include', 'blake2.h') @@ -164,7 +159,6 @@ include_dirs = [] library_dirs = [] define_macros = [] crypto_libraries = ['crypto'] -compression_libraries = ['lz4'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -177,16 +171,17 @@ include_dirs.append(os.path.join(ssl_prefix, 'include')) library_dirs.append(os.path.join(ssl_prefix, 'lib')) -possible_lz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', +possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LZ4_PREFIX'): - possible_lz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) -lz4_prefix = detect_lz4(possible_lz4_prefixes) -if lz4_prefix: - include_dirs.append(os.path.join(lz4_prefix, 'include')) - library_dirs.append(os.path.join(lz4_prefix, 'lib')) -elif not on_rtd: - raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes))) + possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LZ4_PREFIX')) +liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes) +if prefer_system_liblz4 and liblz4_prefix: + print('Detected and preferring liblz4 over bundled LZ4') + define_macros.append(('BORG_USE_LIBLZ4', 'YES')) + liblz4_system = True +else: + liblz4_system = False possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -772,7 +767,10 @@ cmdclass = { ext_modules = [] if not on_rtd: compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, - libraries=compression_libraries, define_macros=define_macros) + define_macros=define_macros) + compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', + system_prefix=liblz4_prefix, system=liblz4_system, + **compress_ext_kwargs) compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', system_prefix=libzstd_prefix, system=libzstd_system, multithreaded=False, legacy=False, **compress_ext_kwargs) diff --git a/setup_lz4.py b/setup_lz4.py new file mode 100644 index 000000000..b3e122046 --- /dev/null +++ b/setup_lz4.py @@ -0,0 +1,72 @@ +# Support code for building a C extension with lz4 files +# +# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) +# 2017-present, Thomas Waldmann (mods to make it more generic, code for lz4) +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os + +# lz4 files, structure as seen in lz4 project repository: + +lz4_sources = [ + 'lib/lz4.c', +] + +lz4_includes = [ + 'lib', +] + + +def lz4_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'lz4.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'LZ4_decompress_safe' in fd.read(): + return prefix + + +def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): + """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization. + + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + use_system = system and system_prefix is not None + + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(lz4_sources, bundled_path) + + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(lz4_includes, bundled_path) + + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['lz4', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if not use_system: + extra_compile_args += [] # not used yet + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/lz4-libselect.h b/src/borg/algorithms/lz4-libselect.h new file mode 100644 index 000000000..85c544d12 --- /dev/null +++ b/src/borg/algorithms/lz4-libselect.h @@ -0,0 +1,5 @@ +#ifdef BORG_USE_LIBLZ4 +#include +#else +#include "lz4/lib/lz4.h" +#endif diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index be2c3c3d1..5fc7f60dd 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -25,9 +25,9 @@ except ImportError: from .helpers import Buffer, DecompressionError -API_VERSION = '1.1_04' +API_VERSION = '1.1_05' -cdef extern from "lz4.h": +cdef extern from "algorithms/lz4-libselect.h": int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil int LZ4_compressBound(int inputSize) nogil diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 3aea2833d..82fd39ff4 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -135,7 +135,7 @@ def check_extension_modules(): raise ExtensionModuleError if chunker.API_VERSION != '1.1_01': raise ExtensionModuleError - if compress.API_VERSION != '1.1_04': + if compress.API_VERSION != '1.1_05': raise ExtensionModuleError if borg.crypto.low_level.API_VERSION != '1.1_02': raise ExtensionModuleError