mirror of https://github.com/borgbackup/borg.git
Merge pull request #4437 from ThomasWaldmann/pkg-config
use pkg-config / refactor setup code
This commit is contained in:
commit
5070401a3d
|
@ -4,26 +4,29 @@ set -e
|
|||
set -x
|
||||
|
||||
if [[ "$(uname -s)" == 'Darwin' ]]; then
|
||||
# HOMEBREW_NO_AUTO_UPDATE=1
|
||||
export HOMEBREW_NO_AUTO_UPDATE=1
|
||||
export HOMEBREW_LOGS=~/brew-logs
|
||||
export HOMEBREW_TEMP=~/brew-temp
|
||||
mkdir $HOMEBREW_LOGS
|
||||
mkdir $HOMEBREW_TEMP
|
||||
# brew update
|
||||
if [[ "${OPENSSL}" != "0.9.8" ]]; then
|
||||
brew outdated openssl || brew upgrade openssl
|
||||
fi
|
||||
brew update > /dev/null
|
||||
brew cleanup > /dev/null # do this here, so it won't automatically trigger in the middle of other stuff
|
||||
brew outdated pkg-config || brew upgrade pkg-config
|
||||
# do NOT update openssl 1.0.x, brew will also update a lot of dependent pkgs (and their dependencies) then!
|
||||
#brew outdated openssl || brew upgrade openssl
|
||||
export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH"
|
||||
brew install readline
|
||||
export PKG_CONFIG_PATH="/usr/local/opt/readline/lib/pkgconfig:$PKG_CONFIG_PATH"
|
||||
brew install zstd
|
||||
brew install lz4
|
||||
brew install xz # required for python lzma module
|
||||
brew install Caskroom/cask/osxfuse
|
||||
|
||||
brew outdated pyenv || brew upgrade pyenv
|
||||
if which pyenv > /dev/null; then
|
||||
eval "$(pyenv init -)"
|
||||
fi
|
||||
|
||||
brew install lz4
|
||||
brew install xz # required for python lzma module
|
||||
brew outdated pyenv || brew upgrade pyenv
|
||||
brew install pkg-config
|
||||
brew install Caskroom/cask/osxfuse
|
||||
|
||||
case "${TOXENV}" in
|
||||
py35)
|
||||
pyenv install 3.5.2
|
||||
|
@ -39,10 +42,11 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
|
|||
else
|
||||
pip install virtualenv
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y fakeroot
|
||||
sudo apt-get install -y liblz4-dev
|
||||
sudo apt-get install -y pkg-config fakeroot
|
||||
#sudo apt-get install -y liblz4-dev # too old on trusty and xenial
|
||||
#sudo apt-get install -y libzstd-dev # too old on trusty and xenial
|
||||
sudo apt-get install -y libacl1-dev
|
||||
sudo apt-get install -y libfuse-dev fuse pkg-config # optional, for FUSE support
|
||||
sudo apt-get install -y libfuse-dev fuse # optional, for FUSE support
|
||||
fi
|
||||
|
||||
python -m virtualenv ~/.venv
|
||||
|
|
|
@ -5,14 +5,11 @@ set -x
|
|||
|
||||
if [[ "$(uname -s)" == "Darwin" ]]; then
|
||||
eval "$(pyenv init -)"
|
||||
if [[ "${OPENSSL}" != "0.9.8" ]]; then
|
||||
# set our flags to use homebrew openssl
|
||||
export ARCHFLAGS="-arch x86_64"
|
||||
export LDFLAGS="-L/usr/local/opt/openssl/lib"
|
||||
export CFLAGS="-I/usr/local/opt/openssl/include"
|
||||
fi
|
||||
# set our flags to use homebrew openssl
|
||||
export ARCHFLAGS="-arch x86_64"
|
||||
export PKG_CONFIG_PATH="/usr/local/opt/openssl/lib/pkgconfig:$PKG_CONFIG_PATH"
|
||||
fi
|
||||
|
||||
# do not use fakeroot, but run as root on travis.
|
||||
# avoids the dreaded EISDIR sporadic failures. see #2482.
|
||||
sudo bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r"
|
||||
sudo -E bash -c "source ~/.venv/bin/activate ; tox -e $TOXENV -r"
|
||||
|
|
|
@ -3,6 +3,7 @@ setuptools_scm
|
|||
pip
|
||||
virtualenv
|
||||
tox
|
||||
pkgconfig
|
||||
pytest
|
||||
pytest-xdist
|
||||
pytest-cov
|
||||
|
|
166
setup.py
166
setup.py
|
@ -1,9 +1,8 @@
|
|||
# borgbackup - main setup code (see also other setup_*.py files)
|
||||
|
||||
import os
|
||||
import io
|
||||
import re
|
||||
import sys
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
from glob import glob
|
||||
|
||||
try:
|
||||
|
@ -21,19 +20,41 @@ try:
|
|||
except ImportError:
|
||||
cythonize = None
|
||||
|
||||
import setup_lz4
|
||||
import setup_zstd
|
||||
import setup_b2
|
||||
import setup_compress
|
||||
import setup_crypto
|
||||
import setup_docs
|
||||
|
||||
# True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code
|
||||
prefer_system_liblz4 = True
|
||||
# How the build process finds the system libs / uses the bundled code:
|
||||
#
|
||||
# 1. it will try to use (system) libs (see 1.1. and 1.2.),
|
||||
# except if you use these env vars to force using the bundled code:
|
||||
# BORG_USE_BUNDLED_XXX undefined --> try using system lib
|
||||
# BORG_USE_BUNDLED_XXX=YES --> use the bundled code
|
||||
# Note: do not use =NO, that is not supported!
|
||||
# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
|
||||
# 1.2. if not and pkg-config can locate the lib, the lib located by
|
||||
# pkg-config will be used. We use the pkg-config tool via the pkgconfig
|
||||
# python package, which must be installed before invoking setup.py.
|
||||
# if pkgconfig is not installed, this step is skipped.
|
||||
# 2. if no system lib could be located via 1.1. or 1.2., it will fall back
|
||||
# to using the bundled code.
|
||||
|
||||
# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code
|
||||
prefer_system_libzstd = True
|
||||
# OpenSSL is required as a (system) lib in any case as we do not bundle it.
|
||||
# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.).
|
||||
# needed: openssl >=1.0.2 or >=1.1.0 (or compatible)
|
||||
system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX')
|
||||
|
||||
# True: use the shared libb2 from the system, False: use the bundled blake2 code
|
||||
prefer_system_libb2 = True
|
||||
# needed: blake2 (>= 0.98.1)
|
||||
prefer_system_libb2 = not bool(os.environ.get('BORG_USE_BUNDLED_B2'))
|
||||
system_prefix_libb2 = os.environ.get('BORG_LIBB2_PREFIX')
|
||||
|
||||
# needed: lz4 (>= 1.7.0 / r129)
|
||||
prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4'))
|
||||
system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
|
||||
|
||||
# needed: zstd (>= 1.3.0)
|
||||
prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
|
||||
system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
|
||||
|
||||
cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
|
||||
|
||||
|
@ -100,79 +121,6 @@ else:
|
|||
raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
|
||||
|
||||
|
||||
def detect_openssl(prefixes):
|
||||
for prefix in prefixes:
|
||||
filename = os.path.join(prefix, 'include', 'openssl', 'evp.h')
|
||||
if os.path.exists(filename):
|
||||
with open(filename, 'rb') as fd:
|
||||
if b'PKCS5_PBKDF2_HMAC(' in fd.read():
|
||||
return prefix
|
||||
|
||||
|
||||
include_dirs = []
|
||||
library_dirs = []
|
||||
define_macros = []
|
||||
|
||||
possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl',
|
||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||
if os.environ.get('BORG_OPENSSL_PREFIX'):
|
||||
possible_openssl_prefixes.insert(0, os.environ.get('BORG_OPENSSL_PREFIX'))
|
||||
ssl_prefix = detect_openssl(possible_openssl_prefixes)
|
||||
if not ssl_prefix:
|
||||
raise Exception('Unable to find OpenSSL >= 1.0 headers. (Looked here: {})'.format(', '.join(possible_openssl_prefixes)))
|
||||
include_dirs.append(os.path.join(ssl_prefix, 'include'))
|
||||
library_dirs.append(os.path.join(ssl_prefix, 'lib'))
|
||||
|
||||
|
||||
possible_liblz4_prefixes = ['/usr', '/usr/local', '/usr/local/opt/lz4', '/usr/local/lz4',
|
||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||
if os.environ.get('BORG_LIBLZ4_PREFIX'):
|
||||
possible_liblz4_prefixes.insert(0, os.environ.get('BORG_LIBLZ4_PREFIX'))
|
||||
liblz4_prefix = setup_lz4.lz4_system_prefix(possible_liblz4_prefixes)
|
||||
if prefer_system_liblz4 and liblz4_prefix:
|
||||
print('Detected and preferring liblz4 over bundled LZ4')
|
||||
define_macros.append(('BORG_USE_LIBLZ4', 'YES'))
|
||||
liblz4_system = True
|
||||
else:
|
||||
liblz4_system = False
|
||||
|
||||
possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2',
|
||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||
if os.environ.get('BORG_LIBB2_PREFIX'):
|
||||
possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX'))
|
||||
libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes)
|
||||
if prefer_system_libb2 and libb2_prefix:
|
||||
print('Detected and preferring libb2 over bundled BLAKE2')
|
||||
define_macros.append(('BORG_USE_LIBB2', 'YES'))
|
||||
libb2_system = True
|
||||
else:
|
||||
libb2_system = False
|
||||
|
||||
possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd',
|
||||
'/usr/local/borg', '/opt/local', '/opt/pkg', ]
|
||||
if os.environ.get('BORG_LIBZSTD_PREFIX'):
|
||||
possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX'))
|
||||
libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes)
|
||||
if prefer_system_libzstd and libzstd_prefix:
|
||||
print('Detected and preferring libzstd over bundled ZSTD')
|
||||
define_macros.append(('BORG_USE_LIBZSTD', 'YES'))
|
||||
libzstd_system = True
|
||||
else:
|
||||
libzstd_system = False
|
||||
|
||||
|
||||
with open('README.rst', 'r') as fd:
|
||||
long_description = fd.read()
|
||||
# remove header, but have one \n before first headline
|
||||
start = long_description.find('What is BorgBackup?')
|
||||
assert start >= 0
|
||||
long_description = '\n' + long_description[start:]
|
||||
# remove badges
|
||||
long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
|
||||
# remove unknown directives
|
||||
long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
|
||||
|
||||
|
||||
def rm(file):
|
||||
try:
|
||||
os.unlink(file)
|
||||
|
@ -202,22 +150,37 @@ cmdclass = {
|
|||
|
||||
ext_modules = []
|
||||
if not on_rtd:
|
||||
compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs,
|
||||
define_macros=define_macros)
|
||||
compress_ext_kwargs = setup_lz4.lz4_ext_kwargs(bundled_path='src/borg/algorithms/lz4',
|
||||
system_prefix=liblz4_prefix, system=liblz4_system,
|
||||
**compress_ext_kwargs)
|
||||
compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd',
|
||||
system_prefix=libzstd_prefix, system=libzstd_system,
|
||||
multithreaded=False, legacy=False, **compress_ext_kwargs)
|
||||
crypto_ext_kwargs = dict(sources=[crypto_ll_source, crypto_helpers], libraries=['crypto'],
|
||||
include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros)
|
||||
crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
|
||||
system_prefix=libb2_prefix, system=libb2_system,
|
||||
**crypto_ext_kwargs)
|
||||
|
||||
def members_appended(*ds):
|
||||
result = defaultdict(list)
|
||||
for d in ds:
|
||||
for k, v in d.items():
|
||||
assert isinstance(v, list)
|
||||
result[k].extend(v)
|
||||
return result
|
||||
|
||||
try:
|
||||
import pkgconfig as pc
|
||||
except ImportError:
|
||||
print('Warning: can not import pkgconfig python package.')
|
||||
pc = None
|
||||
|
||||
crypto_ext_kwargs = members_appended(
|
||||
dict(sources=[crypto_ll_source, crypto_helpers]),
|
||||
setup_crypto.crypto_ext_kwargs(pc, system_prefix_openssl),
|
||||
setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2, system_prefix_libb2),
|
||||
)
|
||||
|
||||
compress_ext_kwargs = members_appended(
|
||||
dict(sources=[compress_source]),
|
||||
setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4),
|
||||
setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd,
|
||||
multithreaded=False, legacy=False),
|
||||
)
|
||||
|
||||
ext_modules += [
|
||||
Extension('borg.compress', **compress_ext_kwargs),
|
||||
Extension('borg.crypto.low_level', **crypto_ext_kwargs),
|
||||
Extension('borg.compress', **compress_ext_kwargs),
|
||||
Extension('borg.hashindex', [hashindex_source]),
|
||||
Extension('borg.item', [item_source]),
|
||||
Extension('borg.chunker', [chunker_source]),
|
||||
|
@ -254,6 +217,7 @@ if not on_rtd:
|
|||
cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts)
|
||||
ext_modules = cythonize(ext_modules, **cython_opts)
|
||||
|
||||
|
||||
setup(
|
||||
name='borgbackup',
|
||||
use_scm_version={
|
||||
|
@ -263,7 +227,7 @@ setup(
|
|||
author_email='borgbackup@python.org',
|
||||
url='https://borgbackup.readthedocs.io/',
|
||||
description='Deduplicated, encrypted, authenticated and compressed backups',
|
||||
long_description=long_description,
|
||||
long_description=setup_docs.long_desc_from_readme(),
|
||||
license='BSD',
|
||||
platforms=['Linux', 'MacOS X', 'FreeBSD', 'OpenBSD', 'NetBSD', ],
|
||||
classifiers=[
|
||||
|
|
72
setup_b2.py
72
setup_b2.py
|
@ -1,72 +0,0 @@
|
|||
# Support code for building a C extension with blake2 files
|
||||
#
|
||||
# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
|
||||
# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2)
|
||||
# All rights reserved.
|
||||
#
|
||||
# This software may be modified and distributed under the terms
|
||||
# of the BSD license. See the LICENSE file for details.
|
||||
|
||||
import os
|
||||
|
||||
# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:
|
||||
|
||||
b2_sources = [
|
||||
'ref/blake2b-ref.c',
|
||||
]
|
||||
|
||||
b2_includes = [
|
||||
'ref',
|
||||
]
|
||||
|
||||
|
||||
def b2_system_prefix(prefixes):
|
||||
for prefix in prefixes:
|
||||
filename = os.path.join(prefix, 'include', 'blake2.h')
|
||||
if os.path.exists(filename):
|
||||
with open(filename, 'rb') as fd:
|
||||
if b'blake2b_init' in fd.read():
|
||||
return prefix
|
||||
|
||||
|
||||
def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
|
||||
"""amend kwargs with b2 stuff for a distutils.extension.Extension initialization.
|
||||
|
||||
bundled_path: relative (to this file) path to the bundled library source code files
|
||||
system_prefix: where the system-installed library can be found
|
||||
system: True: use the system-installed shared library, False: use the bundled library code
|
||||
kwargs: distutils.extension.Extension kwargs that should be amended
|
||||
returns: amended kwargs
|
||||
"""
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path, ))) for path in paths]
|
||||
|
||||
use_system = system and system_prefix is not None
|
||||
|
||||
sources = kwargs.get('sources', [])
|
||||
if not use_system:
|
||||
sources += multi_join(b2_sources, bundled_path)
|
||||
|
||||
include_dirs = kwargs.get('include_dirs', [])
|
||||
if use_system:
|
||||
include_dirs += multi_join(['include'], system_prefix)
|
||||
else:
|
||||
include_dirs += multi_join(b2_includes, bundled_path)
|
||||
|
||||
library_dirs = kwargs.get('library_dirs', [])
|
||||
if use_system:
|
||||
library_dirs += multi_join(['lib'], system_prefix)
|
||||
|
||||
libraries = kwargs.get('libraries', [])
|
||||
if use_system:
|
||||
libraries += ['b2', ]
|
||||
|
||||
extra_compile_args = kwargs.get('extra_compile_args', [])
|
||||
if not use_system:
|
||||
extra_compile_args += [] # not used yet
|
||||
|
||||
ret = dict(**kwargs)
|
||||
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
|
||||
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
|
||||
return ret
|
|
@ -0,0 +1,132 @@
|
|||
# Support code for building a C extension with compression code
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path,))) for path in paths]
|
||||
|
||||
|
||||
# zstd files, structure as seen in zstd project repository:
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
zstd_bundled_path = 'src/borg/algorithms/zstd'
|
||||
|
||||
zstd_sources = [
|
||||
'lib/common/debug.c',
|
||||
'lib/common/entropy_common.c',
|
||||
'lib/common/error_private.c',
|
||||
'lib/common/fse_decompress.c',
|
||||
'lib/common/pool.c',
|
||||
'lib/common/threading.c',
|
||||
'lib/common/xxhash.c',
|
||||
'lib/common/zstd_common.c',
|
||||
'lib/compress/fse_compress.c',
|
||||
'lib/compress/hist.c',
|
||||
'lib/compress/huf_compress.c',
|
||||
'lib/compress/zstd_compress.c',
|
||||
'lib/compress/zstd_double_fast.c',
|
||||
'lib/compress/zstd_fast.c',
|
||||
'lib/compress/zstd_lazy.c',
|
||||
'lib/compress/zstd_ldm.c',
|
||||
'lib/compress/zstd_opt.c',
|
||||
'lib/compress/zstdmt_compress.c',
|
||||
'lib/decompress/huf_decompress.c',
|
||||
'lib/decompress/zstd_ddict.c',
|
||||
'lib/decompress/zstd_decompress.c',
|
||||
'lib/decompress/zstd_decompress_block.c',
|
||||
'lib/dictBuilder/cover.c',
|
||||
'lib/dictBuilder/divsufsort.c',
|
||||
'lib/dictBuilder/fastcover.c',
|
||||
'lib/dictBuilder/zdict.c',
|
||||
]
|
||||
|
||||
zstd_sources_legacy = [
|
||||
'lib/deprecated/zbuff_common.c',
|
||||
'lib/deprecated/zbuff_compress.c',
|
||||
'lib/deprecated/zbuff_decompress.c',
|
||||
'lib/legacy/zstd_v01.c',
|
||||
'lib/legacy/zstd_v02.c',
|
||||
'lib/legacy/zstd_v03.c',
|
||||
'lib/legacy/zstd_v04.c',
|
||||
'lib/legacy/zstd_v05.c',
|
||||
'lib/legacy/zstd_v06.c',
|
||||
'lib/legacy/zstd_v07.c',
|
||||
]
|
||||
|
||||
zstd_includes = [
|
||||
'lib',
|
||||
'lib/common',
|
||||
'lib/compress',
|
||||
'lib/decompress',
|
||||
'lib/dictBuilder',
|
||||
]
|
||||
|
||||
zstd_includes_legacy = [
|
||||
'lib/deprecated',
|
||||
'lib/legacy',
|
||||
]
|
||||
|
||||
|
||||
def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['zstd'])
|
||||
|
||||
if pc and pc.installed('libzstd', '>= 1.3.0'):
|
||||
print('Detected and preferring libzstd [via pkg-config]')
|
||||
return pc.parse('libzstd')
|
||||
|
||||
print('Using bundled ZSTD')
|
||||
sources = multi_join(zstd_sources, zstd_bundled_path)
|
||||
if legacy:
|
||||
sources += multi_join(zstd_sources_legacy, zstd_bundled_path)
|
||||
include_dirs = multi_join(zstd_includes, zstd_bundled_path)
|
||||
if legacy:
|
||||
include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path)
|
||||
extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
|
||||
# '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
|
||||
if legacy:
|
||||
extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
|
||||
if multithreaded:
|
||||
extra_compile_args += ['-DZSTD_MULTITHREAD', ]
|
||||
define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs,
|
||||
extra_compile_args=extra_compile_args, define_macros=define_macros)
|
||||
|
||||
|
||||
# lz4 files, structure as seen in lz4 project repository:
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
lz4_bundled_path = 'src/borg/algorithms/lz4'
|
||||
|
||||
lz4_sources = [
|
||||
'lib/lz4.c',
|
||||
]
|
||||
|
||||
lz4_includes = [
|
||||
'lib',
|
||||
]
|
||||
|
||||
|
||||
def lz4_ext_kwargs(pc, prefer_system, system_prefix):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['lz4'])
|
||||
|
||||
if pc and pc.installed('liblz4', '>= 1.7.0'):
|
||||
print('Detected and preferring liblz4 [via pkg-config]')
|
||||
return pc.parse('liblz4')
|
||||
|
||||
print('Using bundled LZ4')
|
||||
sources = multi_join(lz4_sources, lz4_bundled_path)
|
||||
include_dirs = multi_join(lz4_includes, lz4_bundled_path)
|
||||
define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
|
|
@ -0,0 +1,55 @@
|
|||
# Support code for building a C extension with crypto code
|
||||
|
||||
import os
|
||||
|
||||
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path,))) for path in paths]
|
||||
|
||||
|
||||
def crypto_ext_kwargs(pc, system_prefix):
|
||||
if system_prefix:
|
||||
print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['crypto'])
|
||||
|
||||
if pc and pc.exists('libcrypto'):
|
||||
print('Detected OpenSSL [via pkg-config]')
|
||||
return pc.parse('libcrypto')
|
||||
|
||||
raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX')
|
||||
|
||||
|
||||
# b2 files, structure as seen in BLAKE2 (reference implementation) project repository:
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
b2_bundled_path = 'src/borg/algorithms/blake2'
|
||||
|
||||
b2_sources = [
|
||||
'ref/blake2b-ref.c',
|
||||
]
|
||||
|
||||
b2_includes = [
|
||||
'ref',
|
||||
]
|
||||
|
||||
|
||||
def b2_ext_kwargs(pc, prefer_system, system_prefix):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['b2'])
|
||||
|
||||
if pc and pc.installed('libb2', '>= 0.98.1'):
|
||||
print('Detected and preferring libb2 [via pkg-config]')
|
||||
return pc.parse('libb2')
|
||||
|
||||
print('Using bundled BLAKE2')
|
||||
sources = multi_join(b2_sources, b2_bundled_path)
|
||||
include_dirs = multi_join(b2_includes, b2_bundled_path)
|
||||
define_macros = [('BORG_USE_BUNDLED_B2', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
|
|
@ -11,6 +11,20 @@ from datetime import datetime
|
|||
from setuptools import Command
|
||||
|
||||
|
||||
def long_desc_from_readme():
|
||||
with open('README.rst', 'r') as fd:
|
||||
long_description = fd.read()
|
||||
# remove header, but have one \n before first headline
|
||||
start = long_description.find('What is BorgBackup?')
|
||||
assert start >= 0
|
||||
long_description = '\n' + long_description[start:]
|
||||
# remove badges
|
||||
long_description = re.compile(r'^\.\. start-badges.*^\.\. end-badges', re.M | re.S).sub('', long_description)
|
||||
# remove unknown directives
|
||||
long_description = re.compile(r'^\.\. highlight:: \w+$', re.M).sub('', long_description)
|
||||
return long_description
|
||||
|
||||
|
||||
def format_metavar(option):
|
||||
if option.nargs in ('*', '...'):
|
||||
return '[%s...]' % option.metavar
|
||||
|
|
72
setup_lz4.py
72
setup_lz4.py
|
@ -1,72 +0,0 @@
|
|||
# Support code for building a C extension with lz4 files
|
||||
#
|
||||
# Copyright (c) 2016-present, Gregory Szorc (original code for zstd)
|
||||
# 2017-present, Thomas Waldmann (mods to make it more generic, code for lz4)
|
||||
# All rights reserved.
|
||||
#
|
||||
# This software may be modified and distributed under the terms
|
||||
# of the BSD license. See the LICENSE file for details.
|
||||
|
||||
import os
|
||||
|
||||
# lz4 files, structure as seen in lz4 project repository:
|
||||
|
||||
lz4_sources = [
|
||||
'lib/lz4.c',
|
||||
]
|
||||
|
||||
lz4_includes = [
|
||||
'lib',
|
||||
]
|
||||
|
||||
|
||||
def lz4_system_prefix(prefixes):
|
||||
for prefix in prefixes:
|
||||
filename = os.path.join(prefix, 'include', 'lz4.h')
|
||||
if os.path.exists(filename):
|
||||
with open(filename, 'rb') as fd:
|
||||
if b'LZ4_compress_default' in fd.read(): # requires lz4 >= 1.7.0 (r129)
|
||||
return prefix
|
||||
|
||||
|
||||
def lz4_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs):
|
||||
"""amend kwargs with lz4 stuff for a distutils.extension.Extension initialization.
|
||||
|
||||
bundled_path: relative (to this file) path to the bundled library source code files
|
||||
system_prefix: where the system-installed library can be found
|
||||
system: True: use the system-installed shared library, False: use the bundled library code
|
||||
kwargs: distutils.extension.Extension kwargs that should be amended
|
||||
returns: amended kwargs
|
||||
"""
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path, ))) for path in paths]
|
||||
|
||||
use_system = system and system_prefix is not None
|
||||
|
||||
sources = kwargs.get('sources', [])
|
||||
if not use_system:
|
||||
sources += multi_join(lz4_sources, bundled_path)
|
||||
|
||||
include_dirs = kwargs.get('include_dirs', [])
|
||||
if use_system:
|
||||
include_dirs += multi_join(['include'], system_prefix)
|
||||
else:
|
||||
include_dirs += multi_join(lz4_includes, bundled_path)
|
||||
|
||||
library_dirs = kwargs.get('library_dirs', [])
|
||||
if use_system:
|
||||
library_dirs += multi_join(['lib'], system_prefix)
|
||||
|
||||
libraries = kwargs.get('libraries', [])
|
||||
if use_system:
|
||||
libraries += ['lz4', ]
|
||||
|
||||
extra_compile_args = kwargs.get('extra_compile_args', [])
|
||||
if not use_system:
|
||||
extra_compile_args += [] # not used yet
|
||||
|
||||
ret = dict(**kwargs)
|
||||
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
|
||||
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
|
||||
return ret
|
130
setup_zstd.py
130
setup_zstd.py
|
@ -1,130 +0,0 @@
|
|||
# Support code for building a C extension with zstd files
|
||||
#
|
||||
# Copyright (c) 2016-present, Gregory Szorc
|
||||
# 2017-present, Thomas Waldmann (mods to make it more generic)
|
||||
# All rights reserved.
|
||||
#
|
||||
# This software may be modified and distributed under the terms
|
||||
# of the BSD license. See the LICENSE file for details.
|
||||
|
||||
import os
|
||||
|
||||
# zstd files, structure as seen in zstd project repository:
|
||||
|
||||
zstd_sources = [
|
||||
'lib/common/debug.c',
|
||||
'lib/common/entropy_common.c',
|
||||
'lib/common/error_private.c',
|
||||
'lib/common/fse_decompress.c',
|
||||
'lib/common/pool.c',
|
||||
'lib/common/threading.c',
|
||||
'lib/common/xxhash.c',
|
||||
'lib/common/zstd_common.c',
|
||||
'lib/compress/fse_compress.c',
|
||||
'lib/compress/hist.c',
|
||||
'lib/compress/huf_compress.c',
|
||||
'lib/compress/zstd_compress.c',
|
||||
'lib/compress/zstd_double_fast.c',
|
||||
'lib/compress/zstd_fast.c',
|
||||
'lib/compress/zstd_lazy.c',
|
||||
'lib/compress/zstd_ldm.c',
|
||||
'lib/compress/zstd_opt.c',
|
||||
'lib/compress/zstdmt_compress.c',
|
||||
'lib/decompress/huf_decompress.c',
|
||||
'lib/decompress/zstd_ddict.c',
|
||||
'lib/decompress/zstd_decompress.c',
|
||||
'lib/decompress/zstd_decompress_block.c',
|
||||
'lib/dictBuilder/cover.c',
|
||||
'lib/dictBuilder/divsufsort.c',
|
||||
'lib/dictBuilder/fastcover.c',
|
||||
'lib/dictBuilder/zdict.c',
|
||||
]
|
||||
|
||||
zstd_sources_legacy = [
|
||||
'lib/deprecated/zbuff_common.c',
|
||||
'lib/deprecated/zbuff_compress.c',
|
||||
'lib/deprecated/zbuff_decompress.c',
|
||||
'lib/legacy/zstd_v01.c',
|
||||
'lib/legacy/zstd_v02.c',
|
||||
'lib/legacy/zstd_v03.c',
|
||||
'lib/legacy/zstd_v04.c',
|
||||
'lib/legacy/zstd_v05.c',
|
||||
'lib/legacy/zstd_v06.c',
|
||||
'lib/legacy/zstd_v07.c',
|
||||
]
|
||||
|
||||
zstd_includes = [
|
||||
'lib',
|
||||
'lib/common',
|
||||
'lib/compress',
|
||||
'lib/decompress',
|
||||
'lib/dictBuilder',
|
||||
]
|
||||
|
||||
zstd_includes_legacy = [
|
||||
'lib/deprecated',
|
||||
'lib/legacy',
|
||||
]
|
||||
|
||||
|
||||
def zstd_system_prefix(prefixes):
|
||||
for prefix in prefixes:
|
||||
filename = os.path.join(prefix, 'include', 'zstd.h')
|
||||
if os.path.exists(filename):
|
||||
with open(filename, 'rb') as fd:
|
||||
if b'ZSTD_getFrameContentSize' in fd.read(): # checks for zstd >= 1.3.0
|
||||
return prefix
|
||||
|
||||
|
||||
def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs):
|
||||
"""amend kwargs with zstd suff for a distutils.extension.Extension initialization.
|
||||
|
||||
bundled_path: relative (to this file) path to the bundled library source code files
|
||||
system_prefix: where the system-installed library can be found
|
||||
system: True: use the system-installed shared library, False: use the bundled library code
|
||||
multithreaded: True: define ZSTD_MULTITHREAD
|
||||
legacy: include legacy API support
|
||||
kwargs: distutils.extension.Extension kwargs that should be amended
|
||||
returns: amended kwargs
|
||||
"""
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path, ))) for path in paths]
|
||||
|
||||
use_system = system and system_prefix is not None
|
||||
|
||||
sources = kwargs.get('sources', [])
|
||||
if not use_system:
|
||||
sources += multi_join(zstd_sources, bundled_path)
|
||||
if legacy:
|
||||
sources += multi_join(zstd_sources_legacy, bundled_path)
|
||||
|
||||
include_dirs = kwargs.get('include_dirs', [])
|
||||
if use_system:
|
||||
include_dirs += multi_join(['include'], system_prefix)
|
||||
else:
|
||||
include_dirs += multi_join(zstd_includes, bundled_path)
|
||||
if legacy:
|
||||
include_dirs += multi_join(zstd_includes_legacy, bundled_path)
|
||||
|
||||
library_dirs = kwargs.get('library_dirs', [])
|
||||
if use_system:
|
||||
library_dirs += multi_join(['lib'], system_prefix)
|
||||
|
||||
libraries = kwargs.get('libraries', [])
|
||||
if use_system:
|
||||
libraries += ['zstd', ]
|
||||
|
||||
extra_compile_args = kwargs.get('extra_compile_args', [])
|
||||
if multithreaded:
|
||||
extra_compile_args += ['-DZSTD_MULTITHREAD', ]
|
||||
if not use_system:
|
||||
extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
|
||||
# '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
|
||||
if legacy:
|
||||
extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
|
||||
|
||||
ret = dict(**kwargs)
|
||||
ret.update(dict(sources=sources, extra_compile_args=extra_compile_args,
|
||||
include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries))
|
||||
return ret
|
|
@ -1,5 +1,5 @@
|
|||
#ifdef BORG_USE_LIBB2
|
||||
#include <blake2.h>
|
||||
#else
|
||||
#ifdef BORG_USE_BUNDLED_B2
|
||||
#include "blake2/ref/blake2.h"
|
||||
#else
|
||||
#include <blake2.h>
|
||||
#endif
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#ifdef BORG_USE_LIBLZ4
|
||||
#include <lz4.h>
|
||||
#else
|
||||
#ifdef BORG_USE_BUNDLED_LZ4
|
||||
#include "lz4/lib/lz4.h"
|
||||
#else
|
||||
#include <lz4.h>
|
||||
#endif
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
#ifdef BORG_USE_LIBZSTD
|
||||
#ifdef BORG_USE_BUNDLED_ZSTD
|
||||
#include <zstd.h>
|
||||
#else
|
||||
#include "zstd/lib/zstd.h"
|
||||
#include <zstd.h>
|
||||
#endif
|
||||
|
|
Loading…
Reference in New Issue