merge 1.0-maint into master

# Conflicts:
#	src/borg/crypto.pyx
#	src/borg/hashindex.pyx
#	src/borg/helpers.py
#	src/borg/platform/__init__.py
#	src/borg/platform/darwin.pyx
#	src/borg/platform/freebsd.pyx
#	src/borg/platform/linux.pyx
#	src/borg/remote.py

.travis.yml

@@ -37,8 +37,6 @@ matrix:
           os: osx
           osx_image: xcode6.4
           env: TOXENV=py36
-    allow_failures:
-        - os: osx
 
 install:
     - ./.travis/install.sh

src/borg/archiver.py

@@ -1914,8 +1914,12 @@ class Archiver:
 
         create_epilog = textwrap.dedent("""
         This command creates a backup archive containing all files found while recursively
-        traversing all paths specified. The archive will consume almost no disk space for
-        files or parts of files that have already been stored in other archives.
+        traversing all paths specified. When giving '-' as path, borg will read data
+        from standard input and create a file 'stdin' in the created archive from that
+        data.
+
+        The archive will consume almost no disk space for files or parts of files that
+        have already been stored in other archives.
 
         The archive name needs to be unique. It must not end in '.checkpoint' or
         '.checkpoint.N' (with N being a number), because these names are used for

src/borg/chunker.pyx

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-API_VERSION = 2
+API_VERSION = '1.1_01'
 
 from libc.stdlib cimport free
 

src/borg/compress.pyx

@@ -6,7 +6,7 @@ except ImportError:
 
 from .helpers import Buffer
 
-API_VERSION = 2
+API_VERSION = '1.1_01'
 
 cdef extern from "lz4.h":
     int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil

src/borg/crypto.pyx

@@ -7,7 +7,7 @@ from math import ceil
 from libc.stdlib cimport malloc, free
 from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 
-API_VERSION = 4
+API_VERSION = '1.1_01'
 
 
 cdef extern from "blake2-libselect.h":

src/borg/hashindex.pyx

@@ -8,7 +8,7 @@ from libc.stdint cimport uint32_t, UINT32_MAX, uint64_t
 from libc.errno cimport errno
 from cpython.exc cimport PyErr_SetFromErrnoWithFilename
 
-API_VERSION = 4
+API_VERSION = '1.1_01'
 
 
 cdef extern from "_hashindex.c":

src/borg/helpers.py

@@ -93,17 +93,17 @@ class PlaceholderError(Error):
 
 def check_extension_modules():
     from . import platform, compress, item
-    if hashindex.API_VERSION != 4:
+    if hashindex.API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if chunker.API_VERSION != 2:
+    if chunker.API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if compress.API_VERSION != 2:
+    if compress.API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if crypto.API_VERSION != 4:
+    if crypto.API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if platform.API_VERSION != platform.OS_API_VERSION != 5:
+    if platform.API_VERSION != platform.OS_API_VERSION != '1.1_01':
         raise ExtensionModuleError
-    if item.API_VERSION != 1:
+    if item.API_VERSION != '1.1_01':
         raise ExtensionModuleError
 
 

src/borg/item.pyx

@@ -2,7 +2,7 @@ from .constants import ITEM_KEYS
 from .helpers import safe_encode, safe_decode
 from .helpers import StableDict
 
-API_VERSION = 1
+API_VERSION = '1.1_01'
 
 
 class PropDict:

src/borg/platform/base.py

@@ -13,7 +13,7 @@ platform API: that way platform APIs provided by the platform-specific support m
 are correctly composed into the base functionality.
 """
 
-API_VERSION = 5
+API_VERSION = '1.1_01'
 
 fdatasync = getattr(os, 'fdatasync', os.fsync)
 

src/borg/platform/darwin.pyx

@@ -4,7 +4,7 @@ from ..helpers import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
 from .posix import swidth, umount
 
-API_VERSION = 5
+API_VERSION = '1.1_01'
 
 cdef extern from "sys/acl.h":
     ctypedef struct _acl_t:

src/borg/platform/freebsd.pyx

@@ -4,7 +4,7 @@ from ..helpers import posix_acl_use_stored_uid_gid
 from ..helpers import safe_encode, safe_decode
 from .posix import swidth, umount
 
-API_VERSION = 5
+API_VERSION = '1.1_01'
 
 cdef extern from "errno.h":
     int errno

src/borg/platform/linux.pyx

@@ -13,7 +13,7 @@ from .posix import swidth
 from libc cimport errno
 from libc.stdint cimport int64_t
 
-API_VERSION = 5
+API_VERSION = '1.1_01'
 
 cdef extern from "sys/types.h":
     int ACL_TYPE_ACCESS

src/borg/remote.py

@@ -11,6 +11,7 @@ import tempfile
 import time
 import traceback
 import textwrap
+import time
 from subprocess import Popen, PIPE
 
 import msgpack
@@ -39,6 +40,23 @@ MAX_INFLIGHT = 100
 RATELIMIT_PERIOD = 0.1
 
 
+def os_write(fd, data):
+    """os.write wrapper so we do not lose data for partial writes."""
+    # This is happening frequently on cygwin due to its small pipe buffer size of only 64kiB
+    # and also due to its different blocking pipe behaviour compared to Linux/*BSD.
+    # Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a
+    # signal, in which case serve() would terminate.
+    amount = remaining = len(data)
+    while remaining:
+        count = os.write(fd, data)
+        remaining -= count
+        if not remaining:
+            break
+        data = data[count:]
+        time.sleep(count * 1e-09)
+    return amount
+
+
 class ConnectionClosed(Error):
     """Connection closed by remote host"""
 
@@ -176,7 +194,7 @@ class RepositoryServer:  # pragma: no cover
                     if self.repository is not None:
                         self.repository.close()
                     else:
-                        os.write(stderr_fd, 'Borg {}: Got connection close before repository was opened.\n'
+                        os_write(stderr_fd, 'Borg {}: Got connection close before repository was opened.\n'
                                  .format(__version__).encode())
                     return
                 unpacker.feed(data)
@@ -235,7 +253,7 @@ class RepositoryServer:  # pragma: no cover
                                                     b'exception_short': ex_short,
                                                     b'sysinfo': sysinfo()})
 
-                            os.write(stdout_fd, msg)
+                            os_write(stdout_fd, msg)
                         else:
                             if isinstance(e, (Repository.DoesNotExist, Repository.AlreadyExists, PathNotAllowed)):
                                 # These exceptions are reconstructed on the client end in RemoteRepository.call_many(),
@@ -253,12 +271,12 @@ class RepositoryServer:  # pragma: no cover
                                 logging.error(msg)
                                 logging.log(tb_log_level, tb)
                             exc = 'Remote Exception (see remote log for the traceback)'
-                            os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc)))
+                            os_write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc)))
                     else:
                         if dictFormat:
-                            os.write(stdout_fd, msgpack.packb({MSGID: msgid, RESULT: res}))
+                            os_write(stdout_fd, msgpack.packb({MSGID: msgid, RESULT: res}))
                         else:
-                            os.write(stdout_fd, msgpack.packb((1, msgid, None, res)))
+                            os_write(stdout_fd, msgpack.packb((1, msgid, None, res)))
             if es:
                 self.repository.close()
                 return