Merge pull request #4602 from FelixSchwarz/unbundle-xxhash

ability to unbundle xxhash

setup.py

@@ -20,6 +20,7 @@ try:
 except ImportError:
     cythonize = None
 
+import setup_checksums
 import setup_compress
 import setup_crypto
 import setup_docs
@@ -56,6 +57,9 @@ system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX')
 prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
 system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
 
+prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
+system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
+
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
 
 # Are we building on ReadTheDocs?
@@ -177,13 +181,18 @@ if not on_rtd:
                                        multithreaded=False, legacy=False),
     )
 
+    checksums_ext_kwargs = members_appended(
+        dict(sources=[checksums_source]),
+        setup_checksums.xxhash_ext_kwargs(pc, prefer_system_libxxhash, system_prefix_libxxhash),
+    )
+
     ext_modules += [
         Extension('borg.crypto.low_level', **crypto_ext_kwargs),
         Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.hashindex', [hashindex_source]),
         Extension('borg.item', [item_source]),
         Extension('borg.chunker', [chunker_source]),
-        Extension('borg.algorithms.checksums', [checksums_source]),
+        Extension('borg.algorithms.checksums', **checksums_ext_kwargs),
     ]
 
     posix_ext = Extension('borg.platform.posix', [platform_posix_source])

setup_checksums.py

@@ -0,0 +1,42 @@
+# Support code for building a C extension with checksums code
+
+import os
+
+
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
+# xxhash files, structure as seen in xxhash project repository:
+
+# path relative (to this file) to the bundled library source code files
+xxhash_bundled_path = 'src/borg/algorithms/xxh64'
+
+xxhash_sources = [
+    'xxhash.c',
+]
+
+xxhash_includes = [
+    '',
+]
+
+
+def xxhash_ext_kwargs(pc, prefer_system, system_prefix):
+    if prefer_system:
+        if system_prefix:
+            print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['xxhash'])
+
+        if pc and pc.installed('libxxhash', '>= 1.7.0'):
+            print('Detected and preferring libxxhash [via pkg-config]')
+            return pc.parse('libxxhash')
+
+    print('Using bundled xxhash')
+    sources = multi_join(xxhash_sources, xxhash_bundled_path)
+    include_dirs = multi_join(xxhash_includes, xxhash_bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_XXHASH', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
+

src/borg/algorithms/checksums.pyx

@@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c":
     int _have_clmul "have_clmul"()
 
 
-cdef extern from "xxh64/xxhash.c":
+cdef extern from "xxhash-libselect.h":
     ctypedef struct XXH64_canonical_t:
         char digest[8]
 
@@ -25,6 +25,8 @@ cdef extern from "xxh64/xxhash.c":
         XXH_OK,
         XXH_ERROR
 
+    XXH64_state_t* XXH64_createState();
+    XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
     XXH64_hash_t XXH64(const void* input, size_t length, unsigned long long seed);
 
     XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed);
@@ -80,17 +82,21 @@ def xxh64(data, seed=0):
 
 
 cdef class StreamingXXH64:
-    cdef XXH64_state_t state
+    cdef XXH64_state_t* state
 
     def __cinit__(self, seed=0):
+        self.state = XXH64_createState()
         cdef unsigned long long _seed = seed
-        if XXH64_reset(&self.state, _seed) != XXH_OK:
+        if XXH64_reset(self.state, _seed) != XXH_OK:
             raise Exception('XXH64_reset failed')
 
+    def __dealloc__(self):
+        XXH64_freeState(self.state)
+
     def update(self, data):
         cdef Py_buffer data_buf = ro_buffer(data)
         try:
-            if XXH64_update(&self.state, data_buf.buf, data_buf.len) != XXH_OK:
+            if XXH64_update(self.state, data_buf.buf, data_buf.len) != XXH_OK:
                 raise Exception('XXH64_update failed')
         finally:
             PyBuffer_Release(&data_buf)
@@ -98,7 +104,7 @@ cdef class StreamingXXH64:
     def digest(self):
         cdef XXH64_hash_t hash
         cdef XXH64_canonical_t digest
-        hash = XXH64_digest(&self.state)
+        hash = XXH64_digest(self.state)
         XXH64_canonicalFromHash(&digest, hash)
         return PyBytes_FromStringAndSize(<const char*> digest.digest, 8)
 

src/borg/algorithms/xxhash-libselect.h

@@ -0,0 +1,5 @@
+#ifdef BORG_USE_BUNDLED_XXHASH
+#include "xxh64/xxhash.h"
+#else
+#include <xxhash.h>
+#endif