From 4a2fec32b6429109e9425f1fb54998f3e903a781 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Dec 2017 03:06:12 +0100 Subject: [PATCH] crypto.low_level extension: build blake2 like zstd, see #3415 --- setup.py | 32 ++++++------ setup_b2.py | 72 ++++++++++++++++++++++++++ src/borg/algorithms/blake2-libselect.h | 2 +- 3 files changed, 88 insertions(+), 18 deletions(-) create mode 100644 setup_b2.py diff --git a/setup.py b/setup.py index 5ed7eae1..1a79d558 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import textwrap import setup_lz4 import setup_zstd +import setup_b2 # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code prefer_system_liblz4 = True @@ -21,6 +22,9 @@ prefer_system_liblz4 = True # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code prefer_system_libzstd = True +# True: use the shared libb2 from the system, False: use the bundled blake2 code +prefer_system_libb2 = True + min_python = (3, 5) my_python = sys.version_info @@ -150,19 +154,9 @@ def detect_openssl(prefixes): return prefix -def detect_libb2(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'blake2.h') - if os.path.exists(filename): - with open(filename, 'r') as fd: - if 'blake2b_init' in fd.read(): - return prefix - - include_dirs = [] library_dirs = [] define_macros = [] -crypto_libraries = ['crypto'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -191,13 +185,13 @@ possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/l '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LIBB2_PREFIX'): possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) -libb2_prefix = detect_libb2(possible_libb2_prefixes) -if libb2_prefix: +libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes) +if prefer_system_libb2 and libb2_prefix: print('Detected and preferring libb2 over bundled BLAKE2') - include_dirs.append(os.path.join(libb2_prefix, 'include')) - library_dirs.append(os.path.join(libb2_prefix, 'lib')) - crypto_libraries.append('b2') define_macros.append(('BORG_USE_LIBB2', 'YES')) + libb2_system = True +else: + libb2_system = False possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -778,10 +772,14 @@ if not on_rtd: compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', system_prefix=libzstd_prefix, system=libzstd_system, multithreaded=False, legacy=False, **compress_ext_kwargs) + crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'], + include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) + crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', + system_prefix=libb2_prefix, system=libb2_system, + **crypto_ext_kwargs) ext_modules += [ Extension('borg.compress', **compress_ext_kwargs), - Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, - include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.crypto.low_level', **crypto_ext_kwargs), Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), Extension('borg.chunker', [chunker_source]), diff --git a/setup_b2.py b/setup_b2.py new file mode 100644 index 00000000..e69e0cdf --- /dev/null +++ b/setup_b2.py @@ -0,0 +1,72 @@ +# Support code for building a C extension with blake2 files +# +# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) +# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2) +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os + +# b2 files, structure as seen in BLAKE2 (reference implementation) project repository: + +b2_sources = [ + 'ref/blake2b-ref.c', +] + +b2_includes = [ + 'ref', +] + + +def b2_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'blake2.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'blake2b_init' in fd.read(): + return prefix + + +def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): + """amend kwargs with b2 stuff for a distutils.extension.Extension initialization. + + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + use_system = system and system_prefix is not None + + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(b2_sources, bundled_path) + + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(b2_includes, bundled_path) + + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['b2', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if not use_system: + extra_compile_args += [] # not used yet + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h index 3dbfcd52..5cd970d8 100644 --- a/src/borg/algorithms/blake2-libselect.h +++ b/src/borg/algorithms/blake2-libselect.h @@ -1,5 +1,5 @@ #ifdef BORG_USE_LIBB2 #include #else -#include "blake2/ref/blake2b-ref.c" +#include "blake2/ref/blake2.h" #endif