setup.py: also build a bundled msgpack

based on a patch by felix schwarz, thanks!

changes after that:

build msgpack cpp files at release time, do not use cython otherwise.

we want to include .cpp (and .c) files into the sdist, so users
installing the pypi package won't need to have cython installed.

also: python setup.py clean shall also cleanup the *.cpp and binaries
built from that.

.gitignore: ignore generated msgpack .cpp sources
setup.cfg: flake8: ignore 3rd party bundled msgpack source

.gitignore

@@ -14,6 +14,8 @@ src/borg/platform/darwin.c
 src/borg/platform/freebsd.c
 src/borg/platform/linux.c
 src/borg/platform/posix.c
+src/borg/algorithms/msgpack/_packer.cpp
+src/borg/algorithms/msgpack/_unpacker.cpp
 src/borg/_version.py
 *.egg-info
 *.pyc

setup.cfg

@@ -10,5 +10,5 @@ ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E731,E741,
          W504
 # line length long term target: 120
 max-line-length = 255
-exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py
+exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py,src/borg/algorithms/msgpack
 

setup.py

@@ -39,7 +39,7 @@ install_requires = [
     # we are rather picky about msgpack versions, because a good working msgpack is
     # very important for borg, see https://github.com/borgbackup/borg/issues/3753
     # best versions seem to be 0.4.6, 0.4.7, 0.4.8 and 0.5.6:
-    'msgpack-python >=0.4.6, <=0.5.6, !=0.5.0, !=0.5.1, !=0.5.2, !=0.5.3, !=0.5.4, !=0.5.5',
+    #'msgpack-python >=0.4.6, <=0.5.6, !=0.5.0, !=0.5.1, !=0.5.2, !=0.5.3, !=0.5.4, !=0.5.5',
     # if you can't satisfy the above requirement, these are versions that might
     # also work ok, IF you make sure to use the COMPILED version of msgpack-python,
     # NOT the PURE PYTHON fallback implementation: ==0.5.1, ==0.5.4
@@ -88,8 +88,11 @@ platform_posix_source = 'src/borg/platform/posix.pyx'
 platform_linux_source = 'src/borg/platform/linux.pyx'
 platform_darwin_source = 'src/borg/platform/darwin.pyx'
 platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
+msgpack_packer_source = 'src/borg/algorithms/msgpack/_packer.pyx'
+msgpack_unpacker_source = 'src/borg/algorithms/msgpack/_unpacker.pyx'
 
-cython_sources = [
+cython_c_sources = [
+    # these .pyx will get compiled to .c
     compress_source,
     crypto_ll_source,
     chunker_source,
@@ -103,14 +106,22 @@ cython_sources = [
     platform_darwin_source,
 ]
 
+cython_cpp_sources = [
+    # these .pyx will get compiled to .cpp
+    msgpack_packer_source,
+    msgpack_unpacker_source,
+]
+
 try:
     from Cython.Distutils import build_ext
     import Cython.Compiler.Main as cython_compiler
 
     class Sdist(sdist):
         def __init__(self, *args, **kwargs):
-            for src in cython_sources:
+            for src in cython_c_sources:
                 cython_compiler.compile(src, cython_compiler.default_options)
+            for src in cython_cpp_sources:
+                cython_compiler.compile(src, cplus=True)
             super().__init__(*args, **kwargs)
 
         def make_distribution(self):
@@ -129,6 +140,8 @@ try:
                 'src/borg/platform/linux.c',
                 'src/borg/platform/freebsd.c',
                 'src/borg/platform/darwin.c',
+                'src/borg/algorithms/msgpack/_packer.cpp',
+                'src/borg/algorithms/msgpack/_unpacker.cpp',
             ])
             super().make_distribution()
 
@@ -147,10 +160,15 @@ except ImportError:
     platform_linux_source = platform_linux_source.replace('.pyx', '.c')
     platform_freebsd_source = platform_freebsd_source.replace('.pyx', '.c')
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
+
+    msgpack_packer_source = msgpack_packer_source.replace('.pyx', '.cpp')
+    msgpack_unpacker_source = msgpack_unpacker_source.replace('.pyx', '.cpp')
+
     from distutils.command.build_ext import build_ext
     if not on_rtd and not all(os.path.exists(path) for path in [
         compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, checksums_source,
-        platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
+        platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source,
+        msgpack_packer_source, msgpack_unpacker_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
 
@@ -758,9 +776,13 @@ def rm(file):
 class Clean(clean):
     def run(self):
         super().run()
-        for source in cython_sources:
+        for source in cython_c_sources:
             genc = source.replace('.pyx', '.c')
             rm(genc)
+        for source in cython_cpp_sources:
+            gencpp = source.replace('.pyx', '.cpp')
+            rm(gencpp)
+        for source in cython_c_sources + cython_cpp_sources:
             compiled_glob = source.replace('.pyx', '.cpython*')
             for compiled in sorted(glob(compiled_glob)):
                 rm(compiled)
@@ -788,7 +810,27 @@ if not on_rtd:
     crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2',
                                                system_prefix=libb2_prefix, system=libb2_system,
                                                **crypto_ext_kwargs)
+
+    msgpack_endian = '__BIG_ENDIAN__' if (sys.byteorder == 'big') else '__LITTLE_ENDIAN__'
+    msgpack_macros = [(msgpack_endian, '1')]
+    msgpack_packer_ext_kwargs = dict(
+        sources=[msgpack_packer_source],
+        include_dirs=include_dirs,
+        library_dirs=library_dirs,
+        define_macros=msgpack_macros,
+        language='c++',
+    )
+    msgpack_unpacker_ext_kwargs = dict(
+        sources=[msgpack_unpacker_source],
+        include_dirs=include_dirs,
+        library_dirs=library_dirs,
+        define_macros=msgpack_macros,
+        language='c++',
+    )
+
     ext_modules += [
+        Extension('borg.algorithms.msgpack._packer', **msgpack_packer_ext_kwargs),
+        Extension('borg.algorithms.msgpack._unpacker', **msgpack_unpacker_ext_kwargs),
         Extension('borg.compress', **compress_ext_kwargs),
         Extension('borg.crypto.low_level', **crypto_ext_kwargs),
         Extension('borg.hashindex', [hashindex_source]),