ability to use a system-provided version of "xxhash"

The build process can be controlled via environment variables
similar to other bundled libraries in borgbackup. The main difference
is probably that upstream does not provide a pkgconfig file for
xxhash.

Therefore borg will probably fail to detect the system-provided
version by default (tested on Fedora, seems like Debian and Ubuntu
do not ship a pkgconfig file either). I kept the pkgconfig lookup
code anyway to keep the code as similar as possible to
"setup_compress.py"/"setup_crypto.py".

Setting BORG_LIBXXHASH_PREFIX=/usr helps borg to detect xxhash
on my system (Fedora). You can force the use of the bundled
version of xxhash by setting BORG_USE_BUNDLED_XXHASH=1.

setup.py

@@ -20,6 +20,7 @@
 except ImportError:
     cythonize = None
 
+import setup_checksums
 import setup_compress
 import setup_crypto
 import setup_docs
@@ -56,6 +57,9 @@
 prefer_system_libzstd = not bool(os.environ.get('BORG_USE_BUNDLED_ZSTD'))
 system_prefix_libzstd = os.environ.get('BORG_LIBZSTD_PREFIX')
 
+prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
+system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
+
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
 
 # Are we building on ReadTheDocs?
@@ -178,13 +182,18 @@ def members_appended(*ds):
                                        multithreaded=False, legacy=False),
     )
 
+    checksums_ext_kwargs = members_appended(
+        dict(sources=[checksums_source]),
+        setup_checksums.xxhash_ext_kwargs(pc, prefer_system_libxxhash, system_prefix_libxxhash),
+    )
+
     ext_modules += [
         Extension('borg.crypto.low_level', **crypto_ext_kwargs),
         Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.hashindex', [hashindex_source]),
         Extension('borg.item', [item_source]),
         Extension('borg.chunker', [chunker_source]),
-        Extension('borg.algorithms.checksums', [checksums_source]),
+        Extension('borg.algorithms.checksums', **checksums_ext_kwargs),
     ]
 
     posix_ext = Extension('borg.platform.posix', [platform_posix_source])

setup_checksums.py

@@ -0,0 +1,42 @@
+# Support code for building a C extension with checksums code
+
+import os
+
+
+def multi_join(paths, *path_segments):
+    """apply os.path.join on a list of paths"""
+    return [os.path.join(*(path_segments + (path,))) for path in paths]
+
+
+# xxhash files, structure as seen in xxhash project repository:
+
+# path relative (to this file) to the bundled library source code files
+xxhash_bundled_path = 'src/borg/algorithms/xxh64'
+
+xxhash_sources = [
+    'xxhash.c',
+]
+
+xxhash_includes = [
+    '',
+]
+
+
+def xxhash_ext_kwargs(pc, prefer_system, system_prefix):
+    if prefer_system:
+        if system_prefix:
+            print('Detected and preferring libxxhash [via BORG_LIBXXHASH_PREFIX]')
+            return dict(include_dirs=[os.path.join(system_prefix, 'include')],
+                        library_dirs=[os.path.join(system_prefix, 'lib')],
+                        libraries=['xxhash'])
+
+        if pc and pc.installed('libxxhash', '>= 1.7.0'):
+            print('Detected and preferring libxxhash [via pkg-config]')
+            return pc.parse('libxxhash')
+
+    print('Using bundled xxhash')
+    sources = multi_join(xxhash_sources, xxhash_bundled_path)
+    include_dirs = multi_join(xxhash_includes, xxhash_bundled_path)
+    define_macros = [('BORG_USE_BUNDLED_XXHASH', 'YES')]
+    return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros)
+

src/borg/algorithms/checksums.pyx

@@ -12,7 +12,7 @@ cdef extern from "crc32_dispatch.c":
     int _have_clmul "have_clmul"()
 
 
-cdef extern from "xxh64/xxhash.c":
+cdef extern from "xxhash-libselect.h":
     ctypedef struct XXH64_canonical_t:
         char digest[8]
 

src/borg/algorithms/xxhash-libselect.h

@@ -0,0 +1,5 @@
+#ifdef BORG_USE_BUNDLED_XXHASH
+#include "xxh64/xxhash.h"
+#else
+#include <xxhash.h>
+#endif