From 5173b9f40785ddd3a016de9c838e3d8e55f8dd14 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Mar 2019 05:40:25 +0100 Subject: [PATCH] add pkg-config support, fixes #1925 1. BORG_*_PREFIX is checked (avoids lib detection via pkg-config). 2. pkg-config is tried 3. fallback to bundled C code (or failure in case of OpenSSL) also: - simplified code again - removed (c) headers, nothing left of original code --- .travis/install.sh | 3 +- requirements.d/development.txt | 1 + setup.py | 2 +- setup_b2.py | 64 +++++++------------- setup_crypto.py | 34 ++++------- setup_lz4.py | 62 +++++++------------ setup_zstd.py | 84 ++++++++++---------------- src/borg/algorithms/blake2-libselect.h | 6 +- src/borg/algorithms/lz4-libselect.h | 6 +- src/borg/algorithms/zstd-libselect.h | 4 +- tox.ini | 4 +- 11 files changed, 100 insertions(+), 170 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 5b52cbce..2ae9c20a 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -42,7 +42,8 @@ else sudo apt-get install -y fakeroot sudo apt-get install -y liblz4-dev sudo apt-get install -y libacl1-dev - sudo apt-get install -y libfuse-dev fuse pkg-config # optional, for FUSE support + sudo apt-get install -y pkg-config + sudo apt-get install -y libfuse-dev fuse # optional, for FUSE support fi python -m virtualenv ~/.venv diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 66262e2f..a8f90b87 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -3,6 +3,7 @@ setuptools_scm pip virtualenv tox +pkgconfig pytest pytest-xdist pytest-cov diff --git a/setup.py b/setup.py index 8fcf85f7..30593469 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ prefer_system_liblz4 = True # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code prefer_system_libzstd = True -# True: use the shared libb2 from the system, False: use the bundled blake2 code +# True: use the shared libb2 (>= 0.98.1) from the system, False: use the bundled blake2 code prefer_system_libb2 = True cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1 diff --git a/setup_b2.py b/setup_b2.py index 0f975de4..4b1f5ec8 100644 --- a/setup_b2.py +++ b/setup_b2.py @@ -1,11 +1,4 @@ -# Support code for building a C extension with blake2 files -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with blake2 import os @@ -23,41 +16,28 @@ b2_includes = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def b2_ext_kwargs(prefer_system): - """return kwargs with b2 stuff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBB2_PREFIX') + if system_prefix: + print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['b2']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('libb2', '>= 0.98.1'): + print('Detected and preferring libb2 [via pkg-config]') + return pkgconfig.parse('libb2') - system_prefix = os.environ.get('BORG_LIBB2_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring libb2 over bundled BLAKE2') - define_macros.append(('BORG_USE_LIBB2', 'YES')) - system = True - else: - print('Using bundled BLAKE2') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['b2', ] - else: - sources = multi_join(b2_sources, bundled_path) - include_dirs = multi_join(b2_includes, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + print('Using bundled BLAKE2') + sources = multi_join(b2_sources, bundled_path) + include_dirs = multi_join(b2_includes, bundled_path) + define_macros = [('BORG_USE_BUNDLED_B2', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_crypto.py b/setup_crypto.py index 543a0768..6252c013 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -1,32 +1,20 @@ -# Support code for building a C extension with crypto from OpenSSL -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for openssl) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with crypto from OpenSSL / LibreSSL import os def crypto_ext_kwargs(): - """return kwargs with crypto stuff for a distutils.extension.Extension initialization. - - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] - system_prefix = os.environ.get('BORG_OPENSSL_PREFIX') if system_prefix: - print('Detected system OpenSSL') - else: - raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') + print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['crypto']) - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['crypto', ] + import pkgconfig - return dict(include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + if pkgconfig.exists('libcrypto'): + print('Detected OpenSSL [via pkg-config]') + return pkgconfig.parse('libcrypto') + + raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') diff --git a/setup_lz4.py b/setup_lz4.py index ebb13e34..57401768 100644 --- a/setup_lz4.py +++ b/setup_lz4.py @@ -1,11 +1,4 @@ # Support code for building a C extension with lz4 files -# -# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) -# 2017-present, Thomas Waldmann (mods to make it more generic, code for lz4) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. import os @@ -23,41 +16,28 @@ lz4_includes = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def lz4_ext_kwargs(prefer_system): - """return kwargs with lz4 stuff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') + if system_prefix: + print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['lz4']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('liblz4', '>= 1.7.0'): + print('Detected and preferring liblz4 [via pkg-config]') + return pkgconfig.parse('liblz4') - system_prefix = os.environ.get('BORG_LIBLZ4_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring liblz4 over bundled LZ4') - define_macros.append(('BORG_USE_LIBLZ4', 'YES')) - system = True - else: - print('Using bundled LZ4') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['lz4', ] - else: - sources = multi_join(lz4_sources, bundled_path) - include_dirs = multi_join(lz4_includes, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + print('Using bundled LZ4') + sources = multi_join(lz4_sources, bundled_path) + include_dirs = multi_join(lz4_includes, bundled_path) + define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/setup_zstd.py b/setup_zstd.py index f0b0405a..117aca0b 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -1,11 +1,4 @@ -# Support code for building a C extension with zstd files -# -# Copyright (c) 2016-present, Gregory Szorc -# 2017-present, Thomas Waldmann (mods to make it more generic) -# All rights reserved. -# -# This software may be modified and distributed under the terms -# of the BSD license. See the LICENSE file for details. +# Support code for building a C extension with zstd import os @@ -70,54 +63,39 @@ zstd_includes_legacy = [ ] +def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path,))) for path in paths] + + def zstd_ext_kwargs(prefer_system, multithreaded=False, legacy=False): - """return kwargs with zstd suff for a distutils.extension.Extension initialization. + if prefer_system: + system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') + if system_prefix: + print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]') + return dict(include_dirs=[os.path.join(system_prefix, 'include')], + library_dirs=[os.path.join(system_prefix, 'lib')], + libraries=['zstd']) - prefer_system: prefer the system-installed library (if found) over the bundled C code - multithreaded: True: define ZSTD_MULTITHREAD - legacy: include legacy API support - returns: kwargs for this lib - """ - def multi_join(paths, *path_segments): - """apply os.path.join on a list of paths""" - return [os.path.join(*(path_segments + (path, ))) for path in paths] + import pkgconfig - define_macros = [] + if pkgconfig.installed('libzstd', '>= 1.3.0'): + print('Detected and preferring libzstd [via pkg-config]') + return pkgconfig.parse('libzstd') - system_prefix = os.environ.get('BORG_LIBZSTD_PREFIX') - if prefer_system and system_prefix: - print('Detected and preferring libzstd over bundled ZSTD') - define_macros.append(('BORG_USE_LIBZSTD', 'YES')) - system = True - else: - print('Using bundled ZSTD') - system = False - - use_system = system and system_prefix is not None - - if use_system: - sources = [] - include_dirs = multi_join(['include'], system_prefix) - library_dirs = multi_join(['lib'], system_prefix) - libraries = ['zstd', ] - else: - sources = multi_join(zstd_sources, bundled_path) - if legacy: - sources += multi_join(zstd_sources_legacy, bundled_path) - include_dirs = multi_join(zstd_includes, bundled_path) - if legacy: - include_dirs += multi_join(zstd_includes_legacy, bundled_path) - library_dirs = [] - libraries = [] - - extra_compile_args = [] + print('Using bundled ZSTD') + sources = multi_join(zstd_sources, bundled_path) + if legacy: + sources += multi_join(zstd_sources_legacy, bundled_path) + include_dirs = multi_join(zstd_includes, bundled_path) + if legacy: + include_dirs += multi_join(zstd_includes_legacy, bundled_path) + extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] + # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then + if legacy: + extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] if multithreaded: extra_compile_args += ['-DZSTD_MULTITHREAD', ] - if not use_system: - extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] - # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then - if legacy: - extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] - - return dict(sources=sources, define_macros=define_macros, extra_compile_args=extra_compile_args, - include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries) + define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')] + return dict(sources=sources, include_dirs=include_dirs, + extra_compile_args=extra_compile_args, define_macros=define_macros) diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h index 5cd970d8..e0789e21 100644 --- a/src/borg/algorithms/blake2-libselect.h +++ b/src/borg/algorithms/blake2-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBB2 -#include -#else +#ifdef BORG_USE_BUNDLED_B2 #include "blake2/ref/blake2.h" +#else +#include #endif diff --git a/src/borg/algorithms/lz4-libselect.h b/src/borg/algorithms/lz4-libselect.h index 85c544d1..7abab826 100644 --- a/src/borg/algorithms/lz4-libselect.h +++ b/src/borg/algorithms/lz4-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBLZ4 -#include -#else +#ifdef BORG_USE_BUNDLED_LZ4 #include "lz4/lib/lz4.h" +#else +#include #endif diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h index 9a4ded36..d505823f 100644 --- a/src/borg/algorithms/zstd-libselect.h +++ b/src/borg/algorithms/zstd-libselect.h @@ -1,5 +1,5 @@ -#ifdef BORG_USE_LIBZSTD +#ifdef BORG_USE_BUNDLED_ZSTD #include #else -#include "zstd/lib/zstd.h" +#include #endif diff --git a/tox.ini b/tox.ini index 7a549e9e..2297f228 100644 --- a/tox.ini +++ b/tox.ini @@ -14,5 +14,7 @@ passenv = * [testenv:flake8] changedir = -deps = flake8 +deps = + flake8 + pkgconfig commands = flake8 src scripts conftest.py