From d6635da654808db586f0199a62c6f8124cd77edc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Mar 2019 02:46:00 +0100 Subject: [PATCH 01/16] refactor lib/bundled related code in setup.py - removed hardcoded lib search pathes - to find system libs/headers, one must point these env vars to them: BORG_LIBLZ4_PREFIX, BORG_LIBZSTD_PREFIX, BORG_LIBB2_PREFIX - moved some code from setup.py to setup_*.py --- setup.py | 43 +++---------------------------------------- setup_b2.py | 25 +++++++++++++------------ setup_lz4.py | 25 +++++++++++++------------ setup_zstd.py | 25 +++++++++++++------------ 4 files changed, 42 insertions(+), 76 deletions(-) diff --git a/setup.py b/setup.py index df51ac4a8..ffcac7c0a 100644 --- a/setup.py +++ b/setup.py @@ -124,43 +124,6 @@ include_dirs.append(os.path.join(ssl_prefix, 'include')) library_dirs.append(os.path.join(ssl_prefix, 'lib')) -possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4', - '/usr/local/borg', '/opt/local', '/opt/pkg', ] -if os.environ.get('BORG_LIBLZ4_PREFIX'): - possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX')) -liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes) -if prefer_system_liblz4 and liblz4_prefix: - print('Detected and preferring liblz4 over bundled LZ4') - define_macros.append(('BORG_USE_LIBLZ4', 'YES')) - liblz4_system = True -else: - liblz4_system = False - -possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2', - '/usr/local/borg', '/opt/local', '/opt/pkg', ] -if os.environ.get('BORG_LIBB2_PREFIX'): - possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) -libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes) -if prefer_system_libb2 and libb2_prefix: - print('Detected and preferring libb2 over bundled BLAKE2') - define_macros.append(('BORG_USE_LIBB2', 'YES')) - libb2_system = True -else: - libb2_system = False - -possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd', - '/usr/local/borg', '/opt/local', '/opt/pkg', ] -if os.environ.get('BORG_LIBZSTD_PREFIX'): - possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX')) -libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes) -if prefer_system_libzstd and libzstd_prefix: - print('Detected and preferring libzstd over bundled ZSTD') - define_macros.append(('BORG_USE_LIBZSTD', 'YES')) - libzstd_system = True -else: - libzstd_system = False - - with open('README.rst', 'r') as fd: long_description = fd.read() # remove header, but have one \n before first headline @@ -205,15 +168,15 @@ if not on_rtd: compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', - system_prefix=liblz4_prefix, system=liblz4_system, + prefer_system=prefer_system_liblz4, **compress_ext_kwargs) compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', - system_prefix=libzstd_prefix, system=libzstd_system, + prefer_system=prefer_system_libzstd, multithreaded=False, legacy=False, **compress_ext_kwargs) crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', - system_prefix=libb2_prefix, system=libb2_system, + prefer_system=prefer_system_libb2, **crypto_ext_kwargs) ext_modules += [ Extension('borg.compress', **compress_ext_kwargs), diff --git a/setup_b2.py b/setup_b2.py index e775d9b72..d88bf5cae 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -20,21 +20,11 @@ b2_includes = [ ] -def b2_system_prefix(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'blake2.h') - if os.path.exists(filename): - with open(filename, 'rb') as fd: - if b'blake2b_init' in fd.read(): - return prefix - - -def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): +def b2_ext_kwargs(bundled_path, prefer_system, **kwargs): """amend kwargs with b2 stuff for a distutils.extension.Extension initialization. bundled_path: relative (to this file) path to the bundled library source code files - system_prefix: where the system-installed library can be found - system: True: use the system-installed shared library, False: use the bundled library code + prefer_system: prefer the system-installed library (if found) over the bundled C code kwargs: distutils.extension.Extension kwargs that should be amended returns: amended kwargs """ @@ -42,6 +32,17 @@ def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] + define_macros = kwargs.get('define_macros', []) + + system_prefix = os.environ.get('BORG_LIBB2_PREFIX') + if prefer_system and system_prefix: + print('Detected and preferring libb2 over bundled BLAKE2') + define_macros.append(('BORG_USE_LIBB2', 'YES')) + system = True + else: + print('Using bundled BLAKE2') + system = False + use_system = system and system_prefix is not None sources = kwargs.get('sources', []) diff --git a/setup_lz4.py b/setup_lz4.py index e6d0ddb41..4995e10de 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -20,21 +20,11 @@ lz4_includes = [ ] -def lz4_system_prefix(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'lz4.h') - if os.path.exists(filename): - with open(filename, 'rb') as fd: - if b'LZ4_compress_default' in fd.read(): # requires lz4 >= 1.7.0 (r129) - return prefix - - -def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): +def lz4_ext_kwargs(bundled_path, prefer_system, **kwargs): """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization. bundled_path: relative (to this file) path to the bundled library source code files - system_prefix: where the system-installed library can be found - system: True: use the system-installed shared library, False: use the bundled library code + prefer_system: prefer the system-installed library (if found) over the bundled C code kwargs: distutils.extension.Extension kwargs that should be amended returns: amended kwargs """ @@ -42,6 +32,17 @@ def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] + define_macros = kwargs.get('define_macros', []) + + system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') + if prefer_system and system_prefix: + print('Detected and preferring liblz4 over bundled LZ4') + define_macros.append(('BORG_USE_LIBLZ4', 'YES')) + system = True + else: + print('Using bundled LZ4') + system = False + use_system = system and system_prefix is not None sources = kwargs.get('sources', []) diff --git a/setup_zstd.py b/setup_zstd.py index f024cff90..9615f0dcf 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -67,21 +67,11 @@ zstd_includes_legacy = [ ] -def zstd_system_prefix(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'zstd.h') - if os.path.exists(filename): - with open(filename, 'rb') as fd: - if b'ZSTD_getFrameContentSize' in fd.read(): # checks for zstd >= 1.3.0 - return prefix - - -def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs): +def zstd_ext_kwargs(bundled_path, prefer_system, multithreaded=False, legacy=False, **kwargs): """amend kwargs with zstd suff for a distutils.extension.Extension initialization. bundled_path: relative (to this file) path to the bundled library source code files - system_prefix: where the system-installed library can be found - system: True: use the system-installed shared library, False: use the bundled library code + prefer_system: prefer the system-installed library (if found) over the bundled C code multithreaded: True: define ZSTD_MULTITHREAD legacy: include legacy API support kwargs: distutils.extension.Extension kwargs that should be amended @@ -91,6 +81,17 @@ def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreade """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] + define_macros = kwargs.get('define_macros', []) + + system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') + if prefer_system and system_prefix: + print('Detected and preferring libzstd over bundled ZSTD') + define_macros.append(('BORG_USE_LIBZSTD', 'YES')) + system = True + else: + print('Using bundled ZSTD') + system = False + use_system = system and system_prefix is not None sources = kwargs.get('sources', []) From 2dfc6bb43fbc0024c2a4d3f868d542eebbb960e0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Mar 2019 03:15:59 +0100 Subject: [PATCH 02/16] refactor openssl related code in setup.py - removed hardcoded lib search pathes - to find system libs/headers, one must point these env vars to them: BORG_OPENSSL_PREFIX - moved some code from setup.py to setup_*.py --- setup.py | 33 +++++++-------------------------- setup_crypto.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 26 deletions(-) create mode 100644 setup_crypto.py diff --git a/setup.py b/setup.py index ffcac7c0a..72f6e2a93 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ except ImportError: import setup_lz4 import setup_zstd import setup_b2 +import setup_crypto import setup_docs # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code @@ -100,30 +101,6 @@ else: raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') -def detect_openssl(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'openssl', 'evp.h') - if os.path.exists(filename): - with open(filename, 'rb') as fd: - if b'PKCS5_PBKDF2_HMAC(' in fd.read(): - return prefix - - -include_dirs = [] -library_dirs = [] -define_macros = [] - -possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', - '/usr/local/borg', '/opt/local', '/opt/pkg', ] -if os.environ.get('BORG_OPENSSL_PREFIX'): - possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX')) -ssl_prefix = detect_openssl(possible_openssl_prefixes) -if not ssl_prefix: - raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes))) -include_dirs.append(os.path.join(ssl_prefix, 'include')) -library_dirs.append(os.path.join(ssl_prefix, 'lib')) - - with open('README.rst', 'r') as fd: long_description = fd.read() # remove header, but have one \n before first headline @@ -165,6 +142,9 @@ cmdclass = { ext_modules = [] if not on_rtd: + include_dirs = [] + library_dirs = [] + define_macros = [] compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', @@ -173,8 +153,9 @@ if not on_rtd: compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', prefer_system=prefer_system_libzstd, multithreaded=False, legacy=False, **compress_ext_kwargs) - crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'], - include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) + crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], include_dirs=include_dirs, + library_dirs=library_dirs, define_macros=define_macros) + crypto_ext_kwargs = setup_crypto.crypto_ext_kwargs(**crypto_ext_kwargs) crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', prefer_system=prefer_system_libb2, **crypto_ext_kwargs) diff --git a/setup_crypto.py b/setup_crypto.py new file mode 100644 index 000000000..1da9cfd3f --- /dev/null +++ b/setup_crypto.py @@ -0,0 +1,40 @@ +# Support code for building a C extension with crypto from OpenSSL +# +# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) +# 2017-present, Thomas Waldmann (mods to make it more generic, code for openssl) +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os + + +def crypto_ext_kwargs(**kwargs): + """amend kwargs with crypto stuff for a distutils.extension.Extension initialization. + + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') + if system_prefix: + print('Detected system OpenSSL') + else: + raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') + + include_dirs = kwargs.get('include_dirs', []) + include_dirs += multi_join(['include'], system_prefix) + + library_dirs = kwargs.get('library_dirs', []) + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + libraries += ['crypto', ] + + ret = dict(**kwargs) + ret.update(dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret From 7db29017e5192ec6152bce2d852aae3a7c4380e9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Mar 2019 03:32:32 +0100 Subject: [PATCH 03/16] simplify initial dirs/macros setup --- setup.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 72f6e2a93..71195107e 100644 --- a/setup.py +++ b/setup.py @@ -142,19 +142,14 @@ cmdclass = { ext_modules = [] if not on_rtd: - include_dirs = [] - library_dirs = [] - define_macros = [] - compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, - define_macros=define_macros) + compress_ext_kwargs = dict(sources=[compress_source]) compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', prefer_system=prefer_system_liblz4, **compress_ext_kwargs) compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', prefer_system=prefer_system_libzstd, multithreaded=False, legacy=False, **compress_ext_kwargs) - crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], include_dirs=include_dirs, - library_dirs=library_dirs, define_macros=define_macros) + crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers]) crypto_ext_kwargs = setup_crypto.crypto_ext_kwargs(**crypto_ext_kwargs) crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', prefer_system=prefer_system_libb2, From 24bf01a40b50af9d9ebb82d5673485d096fe386f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Mar 2019 03:44:43 +0100 Subject: [PATCH 04/16] moved bundled_path to setup_*.py --- setup.py | 11 +++-------- setup_b2.py | 6 ++++-- setup_lz4.py | 6 ++++-- setup_zstd.py | 6 ++++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 71195107e..b454c4c8b 100644 --- a/setup.py +++ b/setup.py @@ -143,17 +143,12 @@ cmdclass = { ext_modules = [] if not on_rtd: compress_ext_kwargs = dict(sources=[compress_source]) - compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4', - prefer_system=prefer_system_liblz4, - **compress_ext_kwargs) - compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', - prefer_system=prefer_system_libzstd, + compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(prefer_system_liblz4, **compress_ext_kwargs) + compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, multithreaded=False, legacy=False, **compress_ext_kwargs) crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers]) crypto_ext_kwargs = setup_crypto.crypto_ext_kwargs(**crypto_ext_kwargs) - crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', - prefer_system=prefer_system_libb2, - **crypto_ext_kwargs) + crypto_ext_kwargs = setup_b2.b2_ext_kwargs(prefer_system_libb2, **crypto_ext_kwargs) ext_modules += [ Extension('borg.compress', **compress_ext_kwargs), Extension('borg.crypto.low_level', **crypto_ext_kwargs), diff --git a/setup_b2.py b/setup_b2.py index d88bf5cae..c4eafc5cf 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -11,6 +11,9 @@ import os # b2 files, structure as seen in BLAKE2 (reference implementation) project repository: +# bundled_path: relative (to this file) path to the bundled library source code files +bundled_path = 'src/borg/algorithms/blake2' + b2_sources = [ 'ref/blake2b-ref.c', ] @@ -20,10 +23,9 @@ b2_includes = [ ] -def b2_ext_kwargs(bundled_path, prefer_system, **kwargs): +def b2_ext_kwargs(prefer_system, **kwargs): """amend kwargs with b2 stuff for a distutils.extension.Extension initialization. - bundled_path: relative (to this file) path to the bundled library source code files prefer_system: prefer the system-installed library (if found) over the bundled C code kwargs: distutils.extension.Extension kwargs that should be amended returns: amended kwargs diff --git a/setup_lz4.py b/setup_lz4.py index 4995e10de..271d60c3f 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -11,6 +11,9 @@ import os # lz4 files, structure as seen in lz4 project repository: +# bundled_path: relative (to this file) path to the bundled library source code files +bundled_path = 'src/borg/algorithms/lz4' + lz4_sources = [ 'lib/lz4.c', ] @@ -20,10 +23,9 @@ lz4_includes = [ ] -def lz4_ext_kwargs(bundled_path, prefer_system, **kwargs): +def lz4_ext_kwargs(prefer_system, **kwargs): """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization. - bundled_path: relative (to this file) path to the bundled library source code files prefer_system: prefer the system-installed library (if found) over the bundled C code kwargs: distutils.extension.Extension kwargs that should be amended returns: amended kwargs diff --git a/setup_zstd.py b/setup_zstd.py index 9615f0dcf..a87f8312c 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -11,6 +11,9 @@ import os # zstd files, structure as seen in zstd project repository: +# bundled_path: relative (to this file) path to the bundled library source code files +bundled_path = 'src/borg/algorithms/zstd' + zstd_sources = [ 'lib/common/debug.c', 'lib/common/entropy_common.c', @@ -67,10 +70,9 @@ zstd_includes_legacy = [ ] -def zstd_ext_kwargs(bundled_path, prefer_system, multithreaded=False, legacy=False, **kwargs): +def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False, **kwargs): """amend kwargs with zstd suff for a distutils.extension.Extension initialization. - bundled_path: relative (to this file) path to the bundled library source code files prefer_system: prefer the system-installed library (if found) over the bundled C code multithreaded: True: define ZSTD_MULTITHREAD legacy: include legacy API support From 7b2b8980b4848c298c931898c9093d027403804a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 02:05:24 +0100 Subject: [PATCH 05/16] fix define_macros --- setup_b2.py | 2 +- setup_lz4.py | 2 +- setup_zstd.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup_b2.py b/setup_b2.py index c4eafc5cf..95d2d8db1 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -70,6 +70,6 @@ def b2_ext_kwargs(prefer_system, **kwargs): extra_compile_args += [] # not used yet ret = dict(**kwargs) - ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) return ret diff --git a/setup_lz4.py b/setup_lz4.py index 271d60c3f..40408b29a 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -70,6 +70,6 @@ def lz4_ext_kwargs(prefer_system, **kwargs): extra_compile_args += [] # not used yet ret = dict(**kwargs) - ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) return ret diff --git a/setup_zstd.py b/setup_zstd.py index a87f8312c..2ea2853e4 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -128,6 +128,6 @@ def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False, **kwargs): extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] ret = dict(**kwargs) - ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) return ret From 80ade7093d3dc7e6aa9109409d2e65c2c31a382c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 03:01:00 +0100 Subject: [PATCH 06/16] simplify code by merging kwargs dicts at caller --- setup.py | 29 ++++++++++++++++++++++------- setup_b2.py | 42 +++++++++++++++--------------------------- setup_crypto.py | 22 +++++++--------------- setup_lz4.py | 42 +++++++++++++++--------------------------- setup_zstd.py | 44 +++++++++++++++++--------------------------- 5 files changed, 76 insertions(+), 103 deletions(-) diff --git a/setup.py b/setup.py index b454c4c8b..8fcf85f7b 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,7 @@ import os import io import re import sys +from collections import defaultdict from collections import OrderedDict from datetime import datetime from glob import glob @@ -142,13 +143,27 @@ cmdclass = { ext_modules = [] if not on_rtd: - compress_ext_kwargs = dict(sources=[compress_source]) - compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(prefer_system_liblz4, **compress_ext_kwargs) - compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, - multithreaded=False, legacy=False, **compress_ext_kwargs) - crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers]) - crypto_ext_kwargs = setup_crypto.crypto_ext_kwargs(**crypto_ext_kwargs) - crypto_ext_kwargs = setup_b2.b2_ext_kwargs(prefer_system_libb2, **crypto_ext_kwargs) + + def members_appended(*ds): + result = defaultdict(list) + for d in ds: + for k, v in d.items(): + assert isinstance(v, list) + result[k].extend(v) + return result + + compress_ext_kwargs = members_appended( + dict(sources=[compress_source]), + setup_lz4.lz4_ext_kwargs(prefer_system_liblz4), + setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, multithreaded=False, legacy=False), + ) + + crypto_ext_kwargs = members_appended( + dict(sources=[crypto_ll_source, crypto_helpers]), + setup_crypto.crypto_ext_kwargs(), + setup_b2.b2_ext_kwargs(prefer_system_libb2), + ) + ext_modules += [ Extension('borg.compress', **compress_ext_kwargs), Extension('borg.crypto.low_level', **crypto_ext_kwargs), diff --git a/setup_b2.py b/setup_b2.py index 95d2d8db1..0f975de41 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -23,18 +23,17 @@ b2_includes = [ ] -def b2_ext_kwargs(prefer_system, **kwargs): - """amend kwargs with b2 stuff for a distutils.extension.Extension initialization. +def b2_ext_kwargs(prefer_system): + """return kwargs with b2 stuff for a distutils.extension.Extension initialization. prefer_system: prefer the system-installed library (if found) over the bundled C code - kwargs: distutils.extension.Extension kwargs that should be amended - returns: amended kwargs + returns: kwargs for this lib """ def multi_join(paths, *path_segments): """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] - define_macros = kwargs.get('define_macros', []) + define_macros = [] system_prefix = os.environ.get('BORG_LIBB2_PREFIX') if prefer_system and system_prefix: @@ -47,29 +46,18 @@ def b2_ext_kwargs(prefer_system, **kwargs): use_system = system and system_prefix is not None - sources = kwargs.get('sources', []) - if not use_system: - sources += multi_join(b2_sources, bundled_path) - - include_dirs = kwargs.get('include_dirs', []) if use_system: - include_dirs += multi_join(['include'], system_prefix) + sources = [] + include_dirs = multi_join(['include'], system_prefix) + library_dirs = multi_join(['lib'], system_prefix) + libraries = ['b2', ] else: - include_dirs += multi_join(b2_includes, bundled_path) + sources = multi_join(b2_sources, bundled_path) + include_dirs = multi_join(b2_includes, bundled_path) + library_dirs = [] + libraries = [] - library_dirs = kwargs.get('library_dirs', []) - if use_system: - library_dirs += multi_join(['lib'], system_prefix) + extra_compile_args = [] - libraries = kwargs.get('libraries', []) - if use_system: - libraries += ['b2', ] - - extra_compile_args = kwargs.get('extra_compile_args', []) - if not use_system: - extra_compile_args += [] # not used yet - - ret = dict(**kwargs) - ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) - return ret + return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) diff --git a/setup_crypto.py b/setup_crypto.py index 1da9cfd3f..543a07688 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -10,11 +10,10 @@ import os -def crypto_ext_kwargs(**kwargs): - """amend kwargs with crypto stuff for a distutils.extension.Extension initialization. +def crypto_ext_kwargs(): + """return kwargs with crypto stuff for a distutils.extension.Extension initialization. - kwargs: distutils.extension.Extension kwargs that should be amended - returns: amended kwargs + returns: kwargs for this lib """ def multi_join(paths, *path_segments): """apply os.path.join on a list of paths""" @@ -26,15 +25,8 @@ def crypto_ext_kwargs(**kwargs): else: raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') - include_dirs = kwargs.get('include_dirs', []) - include_dirs += multi_join(['include'], system_prefix) + include_dirs = multi_join(['include'], system_prefix) + library_dirs = multi_join(['lib'], system_prefix) + libraries = ['crypto', ] - library_dirs = kwargs.get('library_dirs', []) - library_dirs += multi_join(['lib'], system_prefix) - - libraries = kwargs.get('libraries', []) - libraries += ['crypto', ] - - ret = dict(**kwargs) - ret.update(dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) - return ret + return dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) diff --git a/setup_lz4.py b/setup_lz4.py index 40408b29a..ebb13e34c 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -23,18 +23,17 @@ lz4_includes = [ ] -def lz4_ext_kwargs(prefer_system, **kwargs): - """amend kwargs with lz4 stuff for a distutils.extension.Extension initialization. +def lz4_ext_kwargs(prefer_system): + """return kwargs with lz4 stuff for a distutils.extension.Extension initialization. prefer_system: prefer the system-installed library (if found) over the bundled C code - kwargs: distutils.extension.Extension kwargs that should be amended - returns: amended kwargs + returns: kwargs for this lib """ def multi_join(paths, *path_segments): """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] - define_macros = kwargs.get('define_macros', []) + define_macros = [] system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') if prefer_system and system_prefix: @@ -47,29 +46,18 @@ def lz4_ext_kwargs(prefer_system, **kwargs): use_system = system and system_prefix is not None - sources = kwargs.get('sources', []) - if not use_system: - sources += multi_join(lz4_sources, bundled_path) - - include_dirs = kwargs.get('include_dirs', []) if use_system: - include_dirs += multi_join(['include'], system_prefix) + sources = [] + include_dirs = multi_join(['include'], system_prefix) + library_dirs = multi_join(['lib'], system_prefix) + libraries = ['lz4', ] else: - include_dirs += multi_join(lz4_includes, bundled_path) + sources = multi_join(lz4_sources, bundled_path) + include_dirs = multi_join(lz4_includes, bundled_path) + library_dirs = [] + libraries = [] - library_dirs = kwargs.get('library_dirs', []) - if use_system: - library_dirs += multi_join(['lib'], system_prefix) + extra_compile_args = [] - libraries = kwargs.get('libraries', []) - if use_system: - libraries += ['lz4', ] - - extra_compile_args = kwargs.get('extra_compile_args', []) - if not use_system: - extra_compile_args += [] # not used yet - - ret = dict(**kwargs) - ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) - return ret + return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) diff --git a/setup_zstd.py b/setup_zstd.py index 2ea2853e4..f0b0405a4 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -70,20 +70,19 @@ zstd_includes_legacy = [ ] -def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False, **kwargs): - """amend kwargs with zstd suff for a distutils.extension.Extension initialization. +def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False): + """return kwargs with zstd suff for a distutils.extension.Extension initialization. prefer_system: prefer the system-installed library (if found) over the bundled C code multithreaded: True: define ZSTD_MULTITHREAD legacy: include legacy API support - kwargs: distutils.extension.Extension kwargs that should be amended - returns: amended kwargs + returns: kwargs for this lib """ def multi_join(paths, *path_segments): """apply os.path.join on a list of paths""" return [os.path.join(*(path_segments + (path, ))) for path in paths] - define_macros = kwargs.get('define_macros', []) + define_macros = [] system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') if prefer_system and system_prefix: @@ -96,29 +95,22 @@ def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False, **kwargs): use_system = system and system_prefix is not None - sources = kwargs.get('sources', []) - if not use_system: - sources += multi_join(zstd_sources, bundled_path) + if use_system: + sources = [] + include_dirs = multi_join(['include'], system_prefix) + library_dirs = multi_join(['lib'], system_prefix) + libraries = ['zstd', ] + else: + sources = multi_join(zstd_sources, bundled_path) if legacy: sources += multi_join(zstd_sources_legacy, bundled_path) - - include_dirs = kwargs.get('include_dirs', []) - if use_system: - include_dirs += multi_join(['include'], system_prefix) - else: - include_dirs += multi_join(zstd_includes, bundled_path) + include_dirs = multi_join(zstd_includes, bundled_path) if legacy: include_dirs += multi_join(zstd_includes_legacy, bundled_path) + library_dirs = [] + libraries = [] - library_dirs = kwargs.get('library_dirs', []) - if use_system: - library_dirs += multi_join(['lib'], system_prefix) - - libraries = kwargs.get('libraries', []) - if use_system: - libraries += ['zstd', ] - - extra_compile_args = kwargs.get('extra_compile_args', []) + extra_compile_args = [] if multithreaded: extra_compile_args += ['-DZSTD_MULTITHREAD', ] if not use_system: @@ -127,7 +119,5 @@ def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False, **kwargs): if legacy: extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] - ret = dict(**kwargs) - ret.update(dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) - return ret + return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) From 5173b9f40785ddd3a016de9c838e3d8e55f8dd14 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 05:40:25 +0100 Subject: [PATCH 07/16] add pkg-config support, fixes #1925 1. BORG_*_PREFIX is checked (avoids lib detection via pkg-config). 2. pkg-config is tried 3. fallback to bundled C code (or failure in case of OpenSSL) also: - simplified code again - removed (c) headers, nothing left of original code --- .travis/install.sh | 3 +- requirements.d/development.txt | 1 + setup.py | 2 +- setup_b2.py | 64 +++++++------------- setup_crypto.py | 34 ++++------- setup_lz4.py | 62 +++++++------------ setup_zstd.py | 84 ++++++++++---------------- src/borg/algorithms/blake2-libselect.h | 6 +- src/borg/algorithms/lz4-libselect.h | 6 +- src/borg/algorithms/zstd-libselect.h | 4 +- tox.ini | 4 +- 11 files changed, 100 insertions(+), 170 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 5b52cbcec..2ae9c20a0 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -42,7 +42,8 @@ else sudo apt-get install -y fakeroot sudo apt-get install -y liblz4-dev sudo apt-get install -y libacl1-dev - sudo apt-get install -y libfuse-dev fuse pkg-config # optional, for FUSE support + sudo apt-get install -y pkg-config + sudo apt-get install -y libfuse-dev fuse # optional, for FUSE support fi python -m virtualenv ~/.venv diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 66262e2f1..a8f90b878 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -3,6 +3,7 @@ setuptools_scm pip virtualenv tox +pkgconfig pytest pytest-xdist pytest-cov diff --git a/setup.py b/setup.py index 8fcf85f7b..305934694 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ prefer_system_liblz4 = True # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code prefer_system_libzstd = True -# True: use the shared libb2 from the system, False: use the bundled blake2 code +# True: use the shared libb2 (>= 0.98.1) from the system, False: use the bundled blake2 code prefer_system_libb2 = True cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 diff --git a/setup_b2.py b/setup_b2.py index 0f975de41..4b1f5ec8c 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -1,11 +1,4 @@ -# Support code for building a C extension with blake2 files -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with blake2 import os @@ -23,41 +16,28 @@ b2_includes = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def b2_ext_kwargs(prefer_system): - """return kwargs with b2 stuff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBB2_PREFIX') + if system_prefix: + print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['b2']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('libb2', '>= 0.98.1'): + print('Detected and preferring libb2 [via pkg-config]') + return pkgconfig.parse('libb2') - system_prefix = os.environ.get('BORG_LIBB2_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring libb2 over bundled BLAKE2') - define_macros.append(('BORG_USE_LIBB2', 'YES')) - system = True - else: - print('Using bundled BLAKE2') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['b2', ] - else: - sources = multi_join(b2_sources, bundled_path) - include_dirs = multi_join(b2_includes, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + print('Using bundled BLAKE2') + sources = multi_join(b2_sources, bundled_path) + include_dirs = multi_join(b2_includes, bundled_path) + define_macros = [('BORG_USE_BUNDLED_B2', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_crypto.py b/setup_crypto.py index 543a07688..6252c013e 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -1,32 +1,20 @@ -# Support code for building a C extension with crypto from OpenSSL -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for openssl) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with crypto from OpenSSL / LibreSSL import os def crypto_ext_kwargs(): - """return kwargs with crypto stuff for a distutils.extension.Extension initialization. - - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] - system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') if system_prefix: - print('Detected system OpenSSL') - else: - raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') + print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['crypto']) - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['crypto', ] + import pkgconfig - return dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + if pkgconfig.exists('libcrypto'): + print('Detected OpenSSL [via pkg-config]') + return pkgconfig.parse('libcrypto') + + raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') diff --git a/setup_lz4.py b/setup_lz4.py index ebb13e34c..574017688 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -1,11 +1,4 @@ # Support code for building a C extension with lz4 files -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for lz4) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. import os @@ -23,41 +16,28 @@ lz4_includes = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def lz4_ext_kwargs(prefer_system): - """return kwargs with lz4 stuff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') + if system_prefix: + print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['lz4']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('liblz4', '>= 1.7.0'): + print('Detected and preferring liblz4 [via pkg-config]') + return pkgconfig.parse('liblz4') - system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring liblz4 over bundled LZ4') - define_macros.append(('BORG_USE_LIBLZ4', 'YES')) - system = True - else: - print('Using bundled LZ4') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['lz4', ] - else: - sources = multi_join(lz4_sources, bundled_path) - include_dirs = multi_join(lz4_includes, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + print('Using bundled LZ4') + sources = multi_join(lz4_sources, bundled_path) + include_dirs = multi_join(lz4_includes, bundled_path) + define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_zstd.py b/setup_zstd.py index f0b0405a4..117aca0b7 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -1,11 +1,4 @@ -# Support code for building a C extension with zstd files -# -# Copyright (c) 2016-present, Gregory Szorc -# 2017-present, Thomas Waldmann (mods to make it more generic) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with zstd import os @@ -70,54 +63,39 @@ zstd_includes_legacy = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False): - """return kwargs with zstd suff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') + if system_prefix: + print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['zstd']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - multithreaded: True: define ZSTD_MULTITHREAD - legacy: include legacy API support - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('libzstd', '>= 1.3.0'): + print('Detected and preferring libzstd [via pkg-config]') + return pkgconfig.parse('libzstd') - system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring libzstd over bundled ZSTD') - define_macros.append(('BORG_USE_LIBZSTD', 'YES')) - system = True - else: - print('Using bundled ZSTD') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['zstd', ] - else: - sources = multi_join(zstd_sources, bundled_path) - if legacy: - sources += multi_join(zstd_sources_legacy, bundled_path) - include_dirs = multi_join(zstd_includes, bundled_path) - if legacy: - include_dirs += multi_join(zstd_includes_legacy, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] + print('Using bundled ZSTD') + sources = multi_join(zstd_sources, bundled_path) + if legacy: + sources += multi_join(zstd_sources_legacy, bundled_path) + include_dirs = multi_join(zstd_includes, bundled_path) + if legacy: + include_dirs += multi_join(zstd_includes_legacy, bundled_path) + extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] + # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then + if legacy: + extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] if multithreaded: extra_compile_args += ['-DZSTD_MULTITHREAD', ] - if not use_system: - extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] - # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then - if legacy: - extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, + extra_compile_args=extra_compile_args, define_macros=define_macros) diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h index 5cd970d88..e0789e218 100644 --- a/src/borg/algorithms/blake2-libselect.h +++ b/src/borg/algorithms/blake2-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBB2 -#include -#else +#ifdef BORG_USE_BUNDLED_B2 #include "blake2/ref/blake2.h" +#else +#include #endif diff --git a/src/borg/algorithms/lz4-libselect.h b/src/borg/algorithms/lz4-libselect.h index 85c544d12..7abab8262 100644 --- a/src/borg/algorithms/lz4-libselect.h +++ b/src/borg/algorithms/lz4-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBLZ4 -#include -#else +#ifdef BORG_USE_BUNDLED_LZ4 #include "lz4/lib/lz4.h" +#else +#include #endif diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h index 9a4ded364..d505823f6 100644 --- a/src/borg/algorithms/zstd-libselect.h +++ b/src/borg/algorithms/zstd-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBZSTD +#ifdef BORG_USE_BUNDLED_ZSTD #include #else -#include "zstd/lib/zstd.h" +#include #endif diff --git a/tox.ini b/tox.ini index 7a549e9e0..2297f2289 100644 --- a/tox.ini +++ b/tox.ini @@ -14,5 +14,7 @@ passenv = * [testenv:flake8] changedir = -deps = flake8 +deps = + flake8 + pkgconfig commands = flake8 src scripts conftest.py From f828e883bdd8fbbb07b3bf6d7cf710c61ce5e8a7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 13:45:19 +0100 Subject: [PATCH 08/16] travis: install script improvements --- .travis/install.sh | 31 +++++++++++++++++-------------- .travis/run.sh | 13 ++++++------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 2ae9c20a0..0bff14cca 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -4,26 +4,29 @@ set -e set -x if [[ "$(uname -s)" == 'Darwin' ]]; then - # HOMEBREW_NO_AUTO_UPDATE=1 + export HOMEBREW_NO_AUTO_UPDATE=1 export HOMEBREW_LOGS=~/brew-logs export HOMEBREW_TEMP=~/brew-temp mkdir $HOMEBREW_LOGS mkdir $HOMEBREW_TEMP - # brew update - if [[ "${OPENSSL}" != "0.9.8" ]]; then - brew outdated openssl || brew upgrade openssl - fi + brew update > /dev/null + brew cleanup > /dev/null # do this here, so it won't automatically trigger in the middle of other stuff + brew outdated pkg-config || brew upgrade pkg-config + # do NOT update openssl 1.0.x, brew will also update a lot of dependent pkgs (and their dependencies) then! + #brew outdated openssl || brew upgrade openssl + export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH" + brew install readline + export PKG_CONFIG_PATH="/usr/local/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH" + brew install zstd + brew install lz4 + brew install xz # required for python lzma module + brew install Caskroom/cask/osxfuse + brew outdated pyenv || brew upgrade pyenv if which pyenv > /dev/null; then eval "$(pyenv init -)" fi - brew install lz4 - brew install xz # required for python lzma module - brew outdated pyenv || brew upgrade pyenv - brew install pkg-config - brew install Caskroom/cask/osxfuse - case "${TOXENV}" in py35) pyenv install 3.5.2 @@ -39,10 +42,10 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then else pip install virtualenv sudo apt-get update - sudo apt-get install -y fakeroot - sudo apt-get install -y liblz4-dev + sudo apt-get install -y pkg-config fakeroot + #sudo apt-get install -y liblz4-dev # too old on trusty and xenial + #sudo apt-get install -y libzstd-dev # too old on trusty and xenial sudo apt-get install -y libacl1-dev - sudo apt-get install -y pkg-config sudo apt-get install -y libfuse-dev fuse # optional, for FUSE support fi diff --git a/.travis/run.sh b/.travis/run.sh index 91f6f7663..e24a11c32 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -5,14 +5,13 @@ set -x if [[ "$(uname -s)" == "Darwin" ]]; then eval "$(pyenv init -)" - if [[ "${OPENSSL}" != "0.9.8" ]]; then - # set our flags to use homebrew openssl - export ARCHFLAGS="-arch x86_64" - export LDFLAGS="-L/usr/local/opt/openssl/lib" - export CFLAGS="-I/usr/local/opt/openssl/include" - fi + # set our flags to use homebrew openssl + export ARCHFLAGS="-arch x86_64" + export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH" + #export LDFLAGS="-L/usr/local/opt/openssl/lib" + #export CFLAGS="-I/usr/local/opt/openssl/include" fi # do not use fakeroot, but run as root on travis. # avoids the dreaded EISDIR sporadic failures. see #2482. -sudo bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r" +sudo -E bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r" From 528aee3ca63d80efd0ffc42480d551e09c1bf42a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 14:53:18 +0100 Subject: [PATCH 09/16] extension building: first crypto, then compression (cosmetic) --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 305934694..d3e271fd0 100644 --- a/setup.py +++ b/setup.py @@ -152,21 +152,21 @@ if not on_rtd: result[k].extend(v) return result - compress_ext_kwargs = members_appended( - dict(sources=[compress_source]), - setup_lz4.lz4_ext_kwargs(prefer_system_liblz4), - setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, multithreaded=False, legacy=False), - ) - crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source, crypto_helpers]), setup_crypto.crypto_ext_kwargs(), setup_b2.b2_ext_kwargs(prefer_system_libb2), ) + compress_ext_kwargs = members_appended( + dict(sources=[compress_source]), + setup_lz4.lz4_ext_kwargs(prefer_system_liblz4), + setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, multithreaded=False, legacy=False), + ) + ext_modules += [ - Extension('borg.compress', **compress_ext_kwargs), Extension('borg.crypto.low_level', **crypto_ext_kwargs), + Extension('borg.compress', **compress_ext_kwargs), Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), Extension('borg.chunker', [chunker_source]), From 2fd0cff4e38b3d80445cb1cdadfe731b517a8bfc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 23:33:57 +0100 Subject: [PATCH 10/16] remove already commented cflags / ldflags --- .travis/run.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis/run.sh b/.travis/run.sh index e24a11c32..1d4d20e2b 100755 --- a/.travis/run.sh +++ b/.travis/run.sh @@ -8,8 +8,6 @@ if [[ "$(uname -s)" == "Darwin" ]]; then # set our flags to use homebrew openssl export ARCHFLAGS="-arch x86_64" export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH" - #export LDFLAGS="-L/usr/local/opt/openssl/lib" - #export CFLAGS="-I/usr/local/opt/openssl/include" fi # do not use fakeroot, but run as root on travis. From 1579d58b3e6ee04b265f14087393e8de8e0ad5e6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 23:59:04 +0100 Subject: [PATCH 11/16] cope with pkgconfig ImportError --- setup.py | 14 ++++++++++---- setup_b2.py | 8 +++----- setup_crypto.py | 8 +++----- setup_lz4.py | 8 +++----- setup_zstd.py | 8 +++----- 5 files changed, 22 insertions(+), 24 deletions(-) diff --git a/setup.py b/setup.py index d3e271fd0..f8a73d032 100644 --- a/setup.py +++ b/setup.py @@ -152,16 +152,22 @@ if not on_rtd: result[k].extend(v) return result + try: + import pkgconfig as pc + except ImportError: + print('Warning: can not import pkgconfig python package.') + pc = None + crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source, crypto_helpers]), - setup_crypto.crypto_ext_kwargs(), - setup_b2.b2_ext_kwargs(prefer_system_libb2), + setup_crypto.crypto_ext_kwargs(pc), + setup_b2.b2_ext_kwargs(pc, prefer_system_libb2), ) compress_ext_kwargs = members_appended( dict(sources=[compress_source]), - setup_lz4.lz4_ext_kwargs(prefer_system_liblz4), - setup_zstd.zstd_ext_kwargs(prefer_system_libzstd, multithreaded=False, legacy=False), + setup_lz4.lz4_ext_kwargs(pc, prefer_system_liblz4), + setup_zstd.zstd_ext_kwargs(pc, prefer_system_libzstd, multithreaded=False, legacy=False), ) ext_modules += [ diff --git a/setup_b2.py b/setup_b2.py index 4b1f5ec8c..309cdb101 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -21,7 +21,7 @@ def multi_join(paths, *path_segments): return [os.path.join(*(path_segments + (path,))) for path in paths] -def b2_ext_kwargs(prefer_system): +def b2_ext_kwargs(pc, prefer_system): if prefer_system: system_prefix = os.environ.get('BORG_LIBB2_PREFIX') if system_prefix: @@ -30,11 +30,9 @@ def b2_ext_kwargs(prefer_system): library_dirs=[os.path.join(system_prefix, 'lib')], libraries=['b2']) - import pkgconfig - - if pkgconfig.installed('libb2', '>= 0.98.1'): + if pc and pc.installed('libb2', '>= 0.98.1'): print('Detected and preferring libb2 [via pkg-config]') - return pkgconfig.parse('libb2') + return pc.parse('libb2') print('Using bundled BLAKE2') sources = multi_join(b2_sources, bundled_path) diff --git a/setup_crypto.py b/setup_crypto.py index 6252c013e..b0971ff62 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -3,7 +3,7 @@ import os -def crypto_ext_kwargs(): +def crypto_ext_kwargs(pc): system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') if system_prefix: print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]') @@ -11,10 +11,8 @@ def crypto_ext_kwargs(): library_dirs=[os.path.join(system_prefix, 'lib')], libraries=['crypto']) - import pkgconfig - - if pkgconfig.exists('libcrypto'): + if pc and pc.exists('libcrypto'): print('Detected OpenSSL [via pkg-config]') - return pkgconfig.parse('libcrypto') + return pc.parse('libcrypto') raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') diff --git a/setup_lz4.py b/setup_lz4.py index 574017688..b8b5f7dfb 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -21,7 +21,7 @@ def multi_join(paths, *path_segments): return [os.path.join(*(path_segments + (path,))) for path in paths] -def lz4_ext_kwargs(prefer_system): +def lz4_ext_kwargs(pc, prefer_system): if prefer_system: system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') if system_prefix: @@ -30,11 +30,9 @@ def lz4_ext_kwargs(prefer_system): library_dirs=[os.path.join(system_prefix, 'lib')], libraries=['lz4']) - import pkgconfig - - if pkgconfig.installed('liblz4', '>= 1.7.0'): + if pc and pc.installed('liblz4', '>= 1.7.0'): print('Detected and preferring liblz4 [via pkg-config]') - return pkgconfig.parse('liblz4') + return pc.parse('liblz4') print('Using bundled LZ4') sources = multi_join(lz4_sources, bundled_path) diff --git a/setup_zstd.py b/setup_zstd.py index 117aca0b7..050c72f79 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -68,7 +68,7 @@ def multi_join(paths, *path_segments): return [os.path.join(*(path_segments + (path,))) for path in paths] -def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False): +def zstd_ext_kwargs(pc, prefer_system, multithreaded=False, legacy=False): if prefer_system: system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') if system_prefix: @@ -77,11 +77,9 @@ def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False): library_dirs=[os.path.join(system_prefix, 'lib')], libraries=['zstd']) - import pkgconfig - - if pkgconfig.installed('libzstd', '>= 1.3.0'): + if pc and pc.installed('libzstd', '>= 1.3.0'): print('Detected and preferring libzstd [via pkg-config]') - return pkgconfig.parse('libzstd') + return pc.parse('libzstd') print('Using bundled ZSTD') sources = multi_join(zstd_sources, bundled_path) From b516cf3c3fb24c9f9b98783cda5648b4bc87a45a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Mar 2019 00:15:17 +0100 Subject: [PATCH 12/16] reduce amount of setup_*.py files in toplevel dir --- setup.py | 10 ++--- setup_b2.py | 41 --------------------- setup_zstd.py => setup_compress.py | 59 ++++++++++++++++++++++++------ setup_crypto.py | 41 ++++++++++++++++++++- setup_lz4.py | 41 --------------------- 5 files changed, 91 insertions(+), 101 deletions(-) delete mode 100644 setup_b2.py rename setup_zstd.py => setup_compress.py (64%) delete mode 100644 setup_lz4.py diff --git a/setup.py b/setup.py index f8a73d032..c8c106851 100644 --- a/setup.py +++ b/setup.py @@ -22,9 +22,7 @@ try: except ImportError: cythonize = None -import setup_lz4 -import setup_zstd -import setup_b2 +import setup_compress import setup_crypto import setup_docs @@ -161,13 +159,13 @@ if not on_rtd: crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source, crypto_helpers]), setup_crypto.crypto_ext_kwargs(pc), - setup_b2.b2_ext_kwargs(pc, prefer_system_libb2), + setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2), ) compress_ext_kwargs = members_appended( dict(sources=[compress_source]), - setup_lz4.lz4_ext_kwargs(pc, prefer_system_liblz4), - setup_zstd.zstd_ext_kwargs(pc, prefer_system_libzstd, multithreaded=False, legacy=False), + setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4), + setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, multithreaded=False, legacy=False), ) ext_modules += [ diff --git a/setup_b2.py b/setup_b2.py deleted file mode 100644 index 309cdb101..000000000 --- a/setup_b2.py +++ /dev/null @@ -1,41 +0,0 @@ -# Support code for building a C extension with blake2 - -import os - -# b2 files, structure as seen in BLAKE2 (reference implementation) project repository: - -# bundled_path: relative (to this file) path to the bundled library source code files -bundled_path = 'src/borg/algorithms/blake2' - -b2_sources = [ - 'ref/blake2b-ref.c', -] - -b2_includes = [ - 'ref', -] - - -def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path,))) for path in paths] - - -def b2_ext_kwargs(pc, prefer_system): - if prefer_system: - system_prefix = os.environ.get('BORG_LIBB2_PREFIX') - if system_prefix: - print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['b2']) - - if pc and pc.installed('libb2', '>= 0.98.1'): - print('Detected and preferring libb2 [via pkg-config]') - return pc.parse('libb2') - - print('Using bundled BLAKE2') - sources = multi_join(b2_sources, bundled_path) - include_dirs = multi_join(b2_includes, bundled_path) - define_macros = [('BORG_USE_BUNDLED_B2', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_zstd.py b/setup_compress.py similarity index 64% rename from setup_zstd.py rename to setup_compress.py index 050c72f79..b5c8195ce 100644 --- a/setup_zstd.py +++ b/setup_compress.py @@ -1,11 +1,17 @@ -# Support code for building a C extension with zstd +# Support code for building a C extension with compression code import os + +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + # zstd files, structure as seen in zstd project repository: -# bundled_path: relative (to this file) path to the bundled library source code files -bundled_path = 'src/borg/algorithms/zstd' +# path relative (to this file) to the bundled library source code files +zstd_bundled_path = 'src/borg/algorithms/zstd' zstd_sources = [ 'lib/common/debug.c', @@ -63,11 +69,6 @@ zstd_includes_legacy = [ ] -def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path,))) for path in paths] - - def zstd_ext_kwargs(pc, prefer_system, multithreaded=False, legacy=False): if prefer_system: system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') @@ -82,12 +83,12 @@ def zstd_ext_kwargs(pc, prefer_system, multithreaded=False, legacy=False): return pc.parse('libzstd') print('Using bundled ZSTD') - sources = multi_join(zstd_sources, bundled_path) + sources = multi_join(zstd_sources, zstd_bundled_path) if legacy: - sources += multi_join(zstd_sources_legacy, bundled_path) - include_dirs = multi_join(zstd_includes, bundled_path) + sources += multi_join(zstd_sources_legacy, zstd_bundled_path) + include_dirs = multi_join(zstd_includes, zstd_bundled_path) if legacy: - include_dirs += multi_join(zstd_includes_legacy, bundled_path) + include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path) extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then if legacy: @@ -97,3 +98,37 @@ def zstd_ext_kwargs(pc, prefer_system, multithreaded=False, legacy=False): define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')] return dict(sources=sources, include_dirs=include_dirs, extra_compile_args=extra_compile_args, define_macros=define_macros) + + +# lz4 files, structure as seen in lz4 project repository: + +# path relative (to this file) to the bundled library source code files +lz4_bundled_path = 'src/borg/algorithms/lz4' + +lz4_sources = [ + 'lib/lz4.c', +] + +lz4_includes = [ + 'lib', +] + + +def lz4_ext_kwargs(pc, prefer_system): + if prefer_system: + system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') + if system_prefix: + print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['lz4']) + + if pc and pc.installed('liblz4', '>= 1.7.0'): + print('Detected and preferring liblz4 [via pkg-config]') + return pc.parse('liblz4') + + print('Using bundled LZ4') + sources = multi_join(lz4_sources, lz4_bundled_path) + include_dirs = multi_join(lz4_includes, lz4_bundled_path) + define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_crypto.py b/setup_crypto.py index b0971ff62..600310b18 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -1,8 +1,13 @@ -# Support code for building a C extension with crypto from OpenSSL / LibreSSL +# Support code for building a C extension with crypto code import os +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def crypto_ext_kwargs(pc): system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') if system_prefix: @@ -16,3 +21,37 @@ def crypto_ext_kwargs(pc): return pc.parse('libcrypto') raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') + + +# b2 files, structure as seen in BLAKE2 (reference implementation) project repository: + +# path relative (to this file) to the bundled library source code files +b2_bundled_path = 'src/borg/algorithms/blake2' + +b2_sources = [ + 'ref/blake2b-ref.c', +] + +b2_includes = [ + 'ref', +] + + +def b2_ext_kwargs(pc, prefer_system): + if prefer_system: + system_prefix = os.environ.get('BORG_LIBB2_PREFIX') + if system_prefix: + print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['b2']) + + if pc and pc.installed('libb2', '>= 0.98.1'): + print('Detected and preferring libb2 [via pkg-config]') + return pc.parse('libb2') + + print('Using bundled BLAKE2') + sources = multi_join(b2_sources, b2_bundled_path) + include_dirs = multi_join(b2_includes, b2_bundled_path) + define_macros = [('BORG_USE_BUNDLED_B2', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_lz4.py b/setup_lz4.py deleted file mode 100644 index b8b5f7dfb..000000000 --- a/setup_lz4.py +++ /dev/null @@ -1,41 +0,0 @@ -# Support code for building a C extension with lz4 files - -import os - -# lz4 files, structure as seen in lz4 project repository: - -# bundled_path: relative (to this file) path to the bundled library source code files -bundled_path = 'src/borg/algorithms/lz4' - -lz4_sources = [ - 'lib/lz4.c', -] - -lz4_includes = [ - 'lib', -] - - -def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path,))) for path in paths] - - -def lz4_ext_kwargs(pc, prefer_system): - if prefer_system: - system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') - if system_prefix: - print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['lz4']) - - if pc and pc.installed('liblz4', '>= 1.7.0'): - print('Detected and preferring liblz4 [via pkg-config]') - return pc.parse('liblz4') - - print('Using bundled LZ4') - sources = multi_join(lz4_sources, bundled_path) - include_dirs = multi_join(lz4_includes, bundled_path) - define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) From 3166d145bb811933c74ed4c7ef1d8f15f893bb70 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Mar 2019 00:28:59 +0100 Subject: [PATCH 13/16] setup.py: add comment, remove unused imports --- setup.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index c8c106851..5c02723e7 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,9 @@ +# borgbackup - main setup code (see also other setup_*.py files) + import os -import io import re import sys from collections import defaultdict -from collections import OrderedDict -from datetime import datetime from glob import glob try: From f4e7133a1e705e57c78da58c613c879b32ed9807 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Mar 2019 00:40:30 +0100 Subject: [PATCH 14/16] setup.py: move long_description generation to setup_docs --- setup.py | 16 ++-------------- setup_docs.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index 5c02723e7..c76058936 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ # borgbackup - main setup code (see also other setup_*.py files) import os -import re import sys from collections import defaultdict from glob import glob @@ -99,18 +98,6 @@ else: raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.') -with open('README.rst', 'r') as fd: - long_description = fd.read() - # remove header, but have one \n before first headline - start = long_description.find('What is BorgBackup?') - assert start >= 0 - long_description = '\n' + long_description[start:] - # remove badges - long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description) - # remove unknown directives - long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description) - - def rm(file): try: os.unlink(file) @@ -206,6 +193,7 @@ if not on_rtd: cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts) ext_modules = cythonize(ext_modules, **cython_opts) + setup( name='borgbackup', use_scm_version={ @@ -215,7 +203,7 @@ setup( author_email='borgbackup@python.org', url='https://borgbackup.readthedocs.io/', description='Deduplicated, encrypted, authenticated and compressed backups', - long_description=long_description, + long_description=setup_docs.long_desc_from_readme(), license='BSD', platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ], classifiers=[ diff --git a/setup_docs.py b/setup_docs.py index f29c66c25..ec84a7d0a 100644 --- a/setup_docs.py +++ b/setup_docs.py @@ -11,6 +11,20 @@ from datetime import datetime from setuptools import Command +def long_desc_from_readme(): + with open('README.rst', 'r') as fd: + long_description = fd.read() + # remove header, but have one \n before first headline + start = long_description.find('What is BorgBackup?') + assert start >= 0 + long_description = '\n' + long_description[start:] + # remove badges + long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description) + # remove unknown directives + long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description) + return long_description + + def format_metavar(option): if option.nargs in ('*', '...'): return '[%s...]' % option.metavar From 7b27082a730ee2a38ac2a70433f08f6193080dcd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Mar 2019 19:54:33 +0100 Subject: [PATCH 15/16] setup.py: bundled code vs system libs: use env vars we use anyway these are already used internally when the build system can not find a system library (neither via pkginfo nor BORG_LIBXXX_PREFIX is given) and then triggers usage of the bundled code via these env vars. now they are also used to tell right from the beginning "use the bundled code" and in that case it will not try to locate system libs and headers. --- setup.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index c76058936..48f5c583f 100644 --- a/setup.py +++ b/setup.py @@ -24,14 +24,19 @@ import setup_compress import setup_crypto import setup_docs -# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code -prefer_system_liblz4 = True +# BORG_USE_BUNDLED_XXX=YES --> use the bundled code +# BORG_USE_BUNDLED_XXX undefined --> try using system lib +# Note: do not use =NO, that is not supported! -# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code -prefer_system_libzstd = True +# needed: lz4 (>= 1.7.0 / r129) +prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4')) + +# needed: zstd (>= 1.3.0) +prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD')) + +# needed: blake2 (>= 0.98.1) +prefer_system_libb2 = not bool(os.environ.get('BORG_USE_BUNDLED_B2')) -# True: use the shared libb2 (>= 0.98.1) from the system, False: use the bundled blake2 code -prefer_system_libb2 = True cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 From 426f3752c1cb2466a7b1bb14661521dda326eddc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Mar 2019 21:23:46 +0100 Subject: [PATCH 16/16] lib/header locating: add docs, move env vars querying to setup.py --- setup.py | 43 +++++++++++++++++++++++++++++++------------ setup_compress.py | 6 ++---- setup_crypto.py | 6 ++---- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/setup.py b/setup.py index 48f5c583f..dc5cb248b 100644 --- a/setup.py +++ b/setup.py @@ -24,19 +24,37 @@ import setup_compress import setup_crypto import setup_docs -# BORG_USE_BUNDLED_XXX=YES --> use the bundled code -# BORG_USE_BUNDLED_XXX undefined --> try using system lib -# Note: do not use =NO, that is not supported! +# How the build process finds the system libs / uses the bundled code: +# +# 1. it will try to use (system) libs (see 1.1. and 1.2.), +# except if you use these env vars to force using the bundled code: +# BORG_USE_BUNDLED_XXX undefined --> try using system lib +# BORG_USE_BUNDLED_XXX=YES --> use the bundled code +# Note: do not use =NO, that is not supported! +# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there. +# 1.2. if not and pkg-config can locate the lib, the lib located by +# pkg-config will be used. We use the pkg-config tool via the pkgconfig +# python package, which must be installed before invoking setup.py. +# if pkgconfig is not installed, this step is skipped. +# 2. if no system lib could be located via 1.1. or 1.2., it will fall back +# to using the bundled code. -# needed: lz4 (>= 1.7.0 / r129) -prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4')) - -# needed: zstd (>= 1.3.0) -prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD')) +# OpenSSL is required as a (system) lib in any case as we do not bundle it. +# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.). +# needed: openssl >=1.0.2 or >=1.1.0 (or compatible) +system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX') # needed: blake2 (>= 0.98.1) prefer_system_libb2 = not bool(os.environ.get('BORG_USE_BUNDLED_B2')) +system_prefix_libb2 = os.environ.get('BORG_LIBB2_PREFIX') +# needed: lz4 (>= 1.7.0 / r129) +prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4')) +system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX') + +# needed: zstd (>= 1.3.0) +prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD')) +system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX') cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 @@ -149,14 +167,15 @@ if not on_rtd: crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source, crypto_helpers]), - setup_crypto.crypto_ext_kwargs(pc), - setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2), + setup_crypto.crypto_ext_kwargs(pc, system_prefix_openssl), + setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2, system_prefix_libb2), ) compress_ext_kwargs = members_appended( dict(sources=[compress_source]), - setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4), - setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, multithreaded=False, legacy=False), + setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4), + setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd, + multithreaded=False, legacy=False), ) ext_modules += [ diff --git a/setup_compress.py b/setup_compress.py index b5c8195ce..fd7f129c6 100644 --- a/setup_compress.py +++ b/setup_compress.py @@ -69,9 +69,8 @@ zstd_includes_legacy = [ ] -def zstd_ext_kwargs(pc, prefer_system, multithreaded=False, legacy=False): +def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False): if prefer_system: - system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') if system_prefix: print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]') return dict(include_dirs=[os.path.join(system_prefix, 'include')], @@ -114,9 +113,8 @@ lz4_includes = [ ] -def lz4_ext_kwargs(pc, prefer_system): +def lz4_ext_kwargs(pc, prefer_system, system_prefix): if prefer_system: - system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') if system_prefix: print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') return dict(include_dirs=[os.path.join(system_prefix, 'include')], diff --git a/setup_crypto.py b/setup_crypto.py index 600310b18..e533f40a1 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -8,8 +8,7 @@ def multi_join(paths, *path_segments): return [os.path.join(*(path_segments + (path,))) for path in paths] -def crypto_ext_kwargs(pc): - system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') +def crypto_ext_kwargs(pc, system_prefix): if system_prefix: print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]') return dict(include_dirs=[os.path.join(system_prefix, 'include')], @@ -37,9 +36,8 @@ b2_includes = [ ] -def b2_ext_kwargs(pc, prefer_system): +def b2_ext_kwargs(pc, prefer_system, system_prefix): if prefer_system: - system_prefix = os.environ.get('BORG_LIBB2_PREFIX') if system_prefix: print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') return dict(include_dirs=[os.path.join(system_prefix, 'include')],