mirror of
https://github.com/borgbackup/borg.git
synced 2025-01-03 13:45:31 +00:00
do not support bundled 3rd party code any more, fixes #6316
This commit is contained in:
Thomas Waldmann
parent
159bd06412
commit
96d93dcf0e
9 changed files with 52 additions and 209 deletions
|
|
@ -159,20 +159,18 @@ Dependencies
|
|||
To install Borg from a source package (including pip), you have to install the
|
||||
following dependencies first:
|
||||
|
||||
* `Python 3`_ >= 3.8.0, plus development headers.
|
||||
* OpenSSL_ >= 1.1.1, plus development headers.
|
||||
* libacl_ (which depends on libattr_), both plus development headers.
|
||||
* We have bundled code of the following packages, but borg by default (see
|
||||
setup.py if you want to change that) prefers a shared library if it can
|
||||
be found on the system (lib + dev headers) at build time:
|
||||
* `Python 3`_ >= 3.8.0 (interpreter plus development headers)
|
||||
* Libraries (library plus development headers):
|
||||
|
||||
- OpenSSL_ >= 1.1.1
|
||||
- libacl_ (which depends on libattr_)
|
||||
- liblz4_ >= 1.7.0 (r129)
|
||||
- libzstd_ >= 1.3.0
|
||||
- libxxhash >= 0.8.1 (0.8.0 might work also)
|
||||
* pkg-config (cli tool) and pkgconfig python package (borg uses these to
|
||||
discover header and library location - if it can't import pkgconfig and
|
||||
is not pointed to header/library locations via env vars [see setup.py],
|
||||
it will fall back to using the bundled code, see above).
|
||||
it will raise a fatal error).
|
||||
**These must be present before invoking setup.py!**
|
||||
* some other Python dependencies, pip will automatically install them for you.
|
||||
* optionally, if you wish to mount an archive as a FUSE filesystem, you need
|
||||
|
|
|
|||
35
setup.py
35
setup.py
|
|
@ -26,35 +26,25 @@
|
|||
|
||||
is_win32 = sys.platform.startswith('win32')
|
||||
|
||||
# How the build process finds the system libs / uses the bundled code:
|
||||
# How the build process finds the system libs:
|
||||
#
|
||||
# 1. it will try to use (system) libs (see 1.1. and 1.2.),
|
||||
# except if you use these env vars to force using the bundled code:
|
||||
# BORG_USE_BUNDLED_XXX undefined --> try using system lib
|
||||
# BORG_USE_BUNDLED_XXX=YES --> use the bundled code
|
||||
# Note: do not use =NO, that is not supported!
|
||||
# 1.1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
|
||||
# 1.2. if not and pkg-config can locate the lib, the lib located by
|
||||
# pkg-config will be used. We use the pkg-config tool via the pkgconfig
|
||||
# python package, which must be installed before invoking setup.py.
|
||||
# if pkgconfig is not installed, this step is skipped.
|
||||
# 2. if no system lib could be located via 1.1. or 1.2., it will fall back
|
||||
# to using the bundled code.
|
||||
# 1. if BORG_LIBXXX_PREFIX is set, it will use headers and libs from there.
|
||||
# 2. if not and pkg-config can locate the lib, the lib located by
|
||||
# pkg-config will be used. We use the pkg-config tool via the pkgconfig
|
||||
# python package, which must be installed before invoking setup.py.
|
||||
# if pkgconfig is not installed, this step is skipped.
|
||||
# 3. otherwise raise a fatal error.
|
||||
|
||||
# OpenSSL is required as a (system) lib in any case as we do not bundle it.
|
||||
# Thus, only step 1.1. and 1.2. apply to openssl (but not 1. and 2.).
|
||||
# needed: openssl >=1.0.2 or >=1.1.0 (or compatible)
|
||||
# needed: >=1.1.1 (or compatible)
|
||||
system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX')
|
||||
|
||||
# needed: lz4 (>= 1.7.0 / r129)
|
||||
prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4'))
|
||||
system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
|
||||
|
||||
# needed: zstd (>= 1.3.0)
|
||||
prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
|
||||
system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
|
||||
|
||||
prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
|
||||
# needed: xxhash (>= 0.8.1)
|
||||
system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
|
||||
|
||||
# Number of threads to use for cythonize, not used on windows
|
||||
|
|
@ -187,14 +177,13 @@ def members_appended(*ds):
|
|||
|
||||
compress_ext_kwargs = members_appended(
|
||||
dict(sources=[compress_source]),
|
||||
setup_compress.lz4_ext_kwargs(pc, prefer_system_liblz4, system_prefix_liblz4),
|
||||
setup_compress.zstd_ext_kwargs(pc, prefer_system_libzstd, system_prefix_libzstd,
|
||||
multithreaded=False, legacy=False),
|
||||
setup_compress.lz4_ext_kwargs(pc, system_prefix_liblz4),
|
||||
setup_compress.zstd_ext_kwargs(pc, system_prefix_libzstd),
|
||||
)
|
||||
|
||||
checksums_ext_kwargs = members_appended(
|
||||
dict(sources=[checksums_source]),
|
||||
setup_checksums.xxhash_ext_kwargs(pc, prefer_system_libxxhash, system_prefix_libxxhash),
|
||||
setup_checksums.xxhash_ext_kwargs(pc, system_prefix_libxxhash),
|
||||
)
|
||||
|
||||
ext_modules += [
|
||||
|
|
|
|||
|
|
@ -3,40 +3,15 @@
|
|||
import os
|
||||
|
||||
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path,))) for path in paths]
|
||||
def xxhash_ext_kwargs(pc, system_prefix):
|
||||
if system_prefix:
|
||||
print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['xxhash'])
|
||||
|
||||
if pc and pc.installed('libxxhash', '>= 0.7.3'):
|
||||
print('Detected and preferring libxxhash [via pkg-config]')
|
||||
return pc.parse('libxxhash')
|
||||
|
||||
# xxhash files, structure as seen in xxhash project repository:
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
xxhash_bundled_path = 'src/borg/algorithms/xxh64'
|
||||
|
||||
xxhash_sources = [
|
||||
'xxhash.c',
|
||||
]
|
||||
|
||||
xxhash_includes = [
|
||||
'',
|
||||
]
|
||||
|
||||
|
||||
def xxhash_ext_kwargs(pc, prefer_system, system_prefix):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['xxhash'])
|
||||
|
||||
if pc and pc.installed('libxxhash', '>= 0.7.3'):
|
||||
print('Detected and preferring libxxhash [via pkg-config]')
|
||||
return pc.parse('libxxhash')
|
||||
|
||||
print('Using bundled xxhash')
|
||||
sources = multi_join(xxhash_sources, xxhash_bundled_path)
|
||||
include_dirs = multi_join(xxhash_includes, xxhash_bundled_path)
|
||||
define_macros = [('BORG_USE_BUNDLED_XXHASH', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
|
||||
|
||||
raise Exception('Could not find xxhash lib/headers, please set BORG_LIBXXHASH_PREFIX')
|
||||
|
|
|
|||
|
|
@ -3,133 +3,29 @@
|
|||
import os
|
||||
|
||||
|
||||
def multi_join(paths, *path_segments):
|
||||
"""apply os.path.join on a list of paths"""
|
||||
return [os.path.join(*(path_segments + (path,))) for path in paths]
|
||||
def zstd_ext_kwargs(pc, system_prefix):
|
||||
if system_prefix:
|
||||
print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['zstd'])
|
||||
|
||||
if pc and pc.installed('libzstd', '>= 1.3.0'):
|
||||
print('Detected and preferring libzstd [via pkg-config]')
|
||||
return pc.parse('libzstd')
|
||||
|
||||
raise Exception('Could not find zstd lib/headers, please set BORG_LIBZSTD_PREFIX')
|
||||
|
||||
|
||||
# zstd files, structure as seen in zstd project repository:
|
||||
def lz4_ext_kwargs(pc, system_prefix):
|
||||
if system_prefix:
|
||||
print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['lz4'])
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
zstd_bundled_path = 'src/borg/algorithms/zstd'
|
||||
if pc and pc.installed('liblz4', '>= 1.7.0'):
|
||||
print('Detected and preferring liblz4 [via pkg-config]')
|
||||
return pc.parse('liblz4')
|
||||
|
||||
zstd_sources = [
|
||||
'lib/common/debug.c',
|
||||
'lib/common/entropy_common.c',
|
||||
'lib/common/error_private.c',
|
||||
'lib/common/fse_decompress.c',
|
||||
'lib/common/pool.c',
|
||||
'lib/common/threading.c',
|
||||
'lib/common/xxhash.c',
|
||||
'lib/common/zstd_common.c',
|
||||
'lib/compress/fse_compress.c',
|
||||
'lib/compress/hist.c',
|
||||
'lib/compress/huf_compress.c',
|
||||
'lib/compress/zstd_compress.c',
|
||||
'lib/compress/zstd_compress_literals.c',
|
||||
'lib/compress/zstd_compress_sequences.c',
|
||||
'lib/compress/zstd_compress_superblock.c',
|
||||
'lib/compress/zstd_double_fast.c',
|
||||
'lib/compress/zstd_fast.c',
|
||||
'lib/compress/zstd_lazy.c',
|
||||
'lib/compress/zstd_ldm.c',
|
||||
'lib/compress/zstd_opt.c',
|
||||
'lib/compress/zstdmt_compress.c',
|
||||
'lib/decompress/huf_decompress.c',
|
||||
'lib/decompress/zstd_ddict.c',
|
||||
'lib/decompress/zstd_decompress.c',
|
||||
'lib/decompress/zstd_decompress_block.c',
|
||||
'lib/dictBuilder/cover.c',
|
||||
'lib/dictBuilder/divsufsort.c',
|
||||
'lib/dictBuilder/fastcover.c',
|
||||
'lib/dictBuilder/zdict.c',
|
||||
]
|
||||
|
||||
zstd_sources_legacy = [
|
||||
'lib/deprecated/zbuff_common.c',
|
||||
'lib/deprecated/zbuff_compress.c',
|
||||
'lib/deprecated/zbuff_decompress.c',
|
||||
'lib/legacy/zstd_v01.c',
|
||||
'lib/legacy/zstd_v02.c',
|
||||
'lib/legacy/zstd_v03.c',
|
||||
'lib/legacy/zstd_v04.c',
|
||||
'lib/legacy/zstd_v05.c',
|
||||
'lib/legacy/zstd_v06.c',
|
||||
'lib/legacy/zstd_v07.c',
|
||||
]
|
||||
|
||||
zstd_includes = [
|
||||
'lib',
|
||||
'lib/common',
|
||||
'lib/compress',
|
||||
'lib/decompress',
|
||||
'lib/dictBuilder',
|
||||
]
|
||||
|
||||
zstd_includes_legacy = [
|
||||
'lib/deprecated',
|
||||
'lib/legacy',
|
||||
]
|
||||
|
||||
|
||||
def zstd_ext_kwargs(pc, prefer_system, system_prefix, multithreaded=False, legacy=False):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring libzstd [via BORG_LIBZSTD_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['zstd'])
|
||||
|
||||
if pc and pc.installed('libzstd', '>= 1.3.0'):
|
||||
print('Detected and preferring libzstd [via pkg-config]')
|
||||
return pc.parse('libzstd')
|
||||
|
||||
print('Using bundled ZSTD')
|
||||
sources = multi_join(zstd_sources, zstd_bundled_path)
|
||||
if legacy:
|
||||
sources += multi_join(zstd_sources_legacy, zstd_bundled_path)
|
||||
include_dirs = multi_join(zstd_includes, zstd_bundled_path)
|
||||
if legacy:
|
||||
include_dirs += multi_join(zstd_includes_legacy, zstd_bundled_path)
|
||||
extra_compile_args = ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ]
|
||||
# '-fvisibility=hidden' does not work, doesn't find PyInit_compress then
|
||||
if legacy:
|
||||
extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ]
|
||||
if multithreaded:
|
||||
extra_compile_args += ['-DZSTD_MULTITHREAD', ]
|
||||
define_macros = [('BORG_USE_BUNDLED_ZSTD', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs,
|
||||
extra_compile_args=extra_compile_args, define_macros=define_macros)
|
||||
|
||||
|
||||
# lz4 files, structure as seen in lz4 project repository:
|
||||
|
||||
# path relative (to this file) to the bundled library source code files
|
||||
lz4_bundled_path = 'src/borg/algorithms/lz4'
|
||||
|
||||
lz4_sources = [
|
||||
'lib/lz4.c',
|
||||
]
|
||||
|
||||
lz4_includes = [
|
||||
'lib',
|
||||
]
|
||||
|
||||
|
||||
def lz4_ext_kwargs(pc, prefer_system, system_prefix):
|
||||
if prefer_system:
|
||||
if system_prefix:
|
||||
print('Detected and preferring liblz4 [via BORG_LIBLZ4_PREFIX]')
|
||||
return dict(include_dirs=[os.path.join(system_prefix, 'include')],
|
||||
library_dirs=[os.path.join(system_prefix, 'lib')],
|
||||
libraries=['lz4'])
|
||||
|
||||
if pc and pc.installed('liblz4', '>= 1.7.0'):
|
||||
print('Detected and preferring liblz4 [via pkg-config]')
|
||||
return pc.parse('liblz4')
|
||||
|
||||
print('Using bundled LZ4')
|
||||
sources = multi_join(lz4_sources, lz4_bundled_path)
|
||||
include_dirs = multi_join(lz4_includes, lz4_bundled_path)
|
||||
define_macros = [('BORG_USE_BUNDLED_LZ4', 'YES')]
|
||||
return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
|
||||
raise Exception('Could not find lz4 lib/headers, please set BORG_LIBLZ4_PREFIX')
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c":
|
|||
int _have_clmul "have_clmul"()
|
||||
|
||||
|
||||
cdef extern from "xxhash-libselect.h":
|
||||
cdef extern from "xxhash.h":
|
||||
ctypedef struct XXH64_canonical_t:
|
||||
char digest[8]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
#ifdef BORG_USE_BUNDLED_LZ4
|
||||
#include "lz4/lib/lz4.h"
|
||||
#else
|
||||
#include <lz4.h>
|
||||
#endif
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
#ifdef BORG_USE_BUNDLED_XXHASH
|
||||
#include "xxh64/xxhash.h"
|
||||
#else
|
||||
#include <xxhash.h>
|
||||
#endif
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
#ifdef BORG_USE_BUNDLED_ZSTD
|
||||
#include <zstd.h>
|
||||
#else
|
||||
#include <zstd.h>
|
||||
#endif
|
||||
|
|
@ -30,13 +30,13 @@ from .helpers import Buffer, DecompressionError
|
|||
|
||||
API_VERSION = '1.2_02'
|
||||
|
||||
cdef extern from "algorithms/lz4-libselect.h":
|
||||
cdef extern from "lz4.h":
|
||||
int LZ4_compress_default(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
||||
int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
|
||||
int LZ4_compressBound(int inputSize) nogil
|
||||
|
||||
|
||||
cdef extern from "algorithms/zstd-libselect.h":
|
||||
cdef extern from "zstd.h":
|
||||
size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) nogil
|
||||
size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil
|
||||
size_t ZSTD_compressBound(size_t srcSize) nogil
|
||||
|
|
|
|||
Loading…
Reference in a new issue