Merge pull request #2887 from enkore/f/mt-1a

multithreading: add pyzmq dep, chunker GIL

docs/global.rst.inc

@@ -16,6 +16,7 @@
 .. _libattr: https://savannah.nongnu.org/projects/attr/
 .. _liblz4: https://github.com/Cyan4973/lz4
 .. _libb2: https://github.com/BLAKE2/libb2
+.. _ZeroMQ: http://zeromq.org/
 .. _OpenSSL: https://www.openssl.org/
 .. _`Python 3`: https://www.python.org/
 .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash

docs/installation.rst

@@ -142,6 +142,7 @@ following dependencies first:
 * OpenSSL_ >= 1.0.0, plus development headers.
 * libacl_ (which depends on libattr_), both plus development headers.
 * liblz4_, plus development headers.
+* ZeroMQ_ >= 4.0.0, plus development headers.
 * some Python dependencies, pip will automatically install them for you
 * optionally, the llfuse_ Python package is required if you wish to mount an
   archive as a FUSE filesystem. See setup.py about the version requirements.
@@ -169,6 +170,7 @@ Install the dependencies with development headers::
     libssl-dev openssl \
     libacl1-dev libacl1 \
     liblz4-dev liblz4-1 \
+    libzmq3-dev libzmq3 \
     build-essential
     sudo apt-get install libfuse-dev fuse pkg-config    # optional, for FUSE support
 
@@ -179,6 +181,8 @@ group, log out and log in again.
 Fedora / Korora
 +++++++++++++++
 
+.. todo:: Add zeromq
+
 Install the dependencies with development headers::
 
     sudo dnf install python3 python3-devel python3-pip python3-virtualenv
@@ -192,6 +196,8 @@ Install the dependencies with development headers::
 openSUSE Tumbleweed / Leap
 ++++++++++++++++++++++++++
 
+.. todo:: Add zeromq
+
 Install the dependencies automatically using zypper::
 
     sudo zypper source-install --build-deps-only borgbackup
@@ -207,6 +213,8 @@ Alternatively, you can enumerate all build dependencies in the command line::
 Mac OS X
 ++++++++
 
+.. todo:: Add zeromq
+
 Assuming you have installed homebrew_, the following steps will install all the
 dependencies::
 
@@ -222,7 +230,10 @@ FUSE for OS X, which is available as a pre-release_.
 
 FreeBSD
 ++++++++
-Listed below are packages you will need to install |project_name|, its dependencies,
+
+.. todo:: Add zeromq
+
+Listed below are packages you will need to install Borg, its dependencies,
 and commands to make FUSE work for using the mount command.
 
 ::
@@ -253,6 +264,8 @@ Cygwin
     Running under Cygwin is experimental and has only been tested with Cygwin
     (x86-64) v2.5.2. Remote repositories are known broken, local repositories should work.
 
+.. todo:: Add zeromq
+
 Use the Cygwin installer to install the dependencies::
 
     python3 python3-devel python3-setuptools

setup.py

@@ -22,9 +22,12 @@ if my_python < min_python:
 # Are we building on ReadTheDocs?
 on_rtd = os.environ.get('READTHEDOCS')
 
-# msgpack pure python data corruption was fixed in 0.4.6.
+install_requires = [
-# Also, we might use some rather recent API features.
+    # msgpack pure python data corruption was fixed in 0.4.6.
-install_requires = ['msgpack-python>=0.4.6', ]
+    # Also, we might use some rather recent API features.
+    'msgpack-python>=0.4.6',
+    'pyzmq',
+]
 
 # note for package maintainers: if you package borgbackup for distribution,
 # please add llfuse as a *requirement* on all platforms that have a working

src/borg/_chunker.c

@@ -157,6 +157,8 @@ chunker_fill(Chunker *c)
     off_t offset, length;
     int overshoot;
     PyObject *data;
+    PyThreadState *thread_state;
+
     memmove(c->data, c->data + c->last, c->position + c->remaining - c->last);
     c->position -= c->last;
     c->last = 0;
@@ -165,6 +167,8 @@ chunker_fill(Chunker *c)
         return 1;
     }
     if(c->fh >= 0) {
+        thread_state = PyEval_SaveThread();
+
         offset = c->bytes_read;
         // if we have a os-level file descriptor, use os-level API
         n = read(c->fh, c->data + c->position + c->remaining, n);
@@ -177,6 +181,7 @@ chunker_fill(Chunker *c)
             c->eof = 1;
         }
         else {
+            PyEval_RestoreThread(thread_state);
             // some error happened
             PyErr_SetFromErrno(PyExc_OSError);
             return 0;
@@ -211,6 +216,8 @@ chunker_fill(Chunker *c)
 
         posix_fadvise(c->fh, offset & ~pagemask, length - overshoot, POSIX_FADV_DONTNEED);
         #endif
+
+        PyEval_RestoreThread(thread_state);
     }
     else {
         // no os-level file descriptor, use Python file object API