diff --git a/.travis.yml b/.travis.yml index 3b62d1ebe..4122d5645 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,8 +37,6 @@ matrix: os: osx osx_image: xcode6.4 env: TOXENV=py36 - allow_failures: - - os: osx install: - ./.travis/install.sh diff --git a/src/borg/archiver.py b/src/borg/archiver.py index c7b3b70b5..80ed853cb 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1914,8 +1914,12 @@ class Archiver: create_epilog = textwrap.dedent(""" This command creates a backup archive containing all files found while recursively - traversing all paths specified. The archive will consume almost no disk space for - files or parts of files that have already been stored in other archives. + traversing all paths specified. When giving '-' as path, borg will read data + from standard input and create a file 'stdin' in the created archive from that + data. + + The archive will consume almost no disk space for files or parts of files that + have already been stored in other archives. The archive name needs to be unique. It must not end in '.checkpoint' or '.checkpoint.N' (with N being a number), because these names are used for diff --git a/src/borg/chunker.pyx b/src/borg/chunker.pyx index 560e14c82..bbe47ceca 100644 --- a/src/borg/chunker.pyx +++ b/src/borg/chunker.pyx @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -API_VERSION = 2 +API_VERSION = '1.1_01' from libc.stdlib cimport free diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 50785ea1a..9257ced0b 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -6,7 +6,7 @@ except ImportError: from .helpers import Buffer -API_VERSION = 2 +API_VERSION = '1.1_01' cdef extern from "lz4.h": int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil diff --git a/src/borg/crypto.pyx b/src/borg/crypto.pyx index c3cda4ac3..0ed09141a 100644 --- a/src/borg/crypto.pyx +++ b/src/borg/crypto.pyx @@ -7,7 +7,7 @@ from math import ceil from libc.stdlib cimport malloc, free from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release -API_VERSION = 4 +API_VERSION = '1.1_01' cdef extern from "blake2-libselect.h": diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx index dd224f522..a10d403ae 100644 --- a/src/borg/hashindex.pyx +++ b/src/borg/hashindex.pyx @@ -8,7 +8,7 @@ from libc.stdint cimport uint32_t, UINT32_MAX, uint64_t from libc.errno cimport errno from cpython.exc cimport PyErr_SetFromErrnoWithFilename -API_VERSION = 4 +API_VERSION = '1.1_01' cdef extern from "_hashindex.c": diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 7c71cc919..f4338e75c 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -93,17 +93,17 @@ class PlaceholderError(Error): def check_extension_modules(): from . import platform, compress, item - if hashindex.API_VERSION != 4: + if hashindex.API_VERSION != '1.1_01': raise ExtensionModuleError - if chunker.API_VERSION != 2: + if chunker.API_VERSION != '1.1_01': raise ExtensionModuleError - if compress.API_VERSION != 2: + if compress.API_VERSION != '1.1_01': raise ExtensionModuleError - if crypto.API_VERSION != 4: + if crypto.API_VERSION != '1.1_01': raise ExtensionModuleError - if platform.API_VERSION != platform.OS_API_VERSION != 5: + if platform.API_VERSION != platform.OS_API_VERSION != '1.1_01': raise ExtensionModuleError - if item.API_VERSION != 1: + if item.API_VERSION != '1.1_01': raise ExtensionModuleError diff --git a/src/borg/item.pyx b/src/borg/item.pyx index bdcb9a539..4ac960a63 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -2,7 +2,7 @@ from .constants import ITEM_KEYS from .helpers import safe_encode, safe_decode from .helpers import StableDict -API_VERSION = 1 +API_VERSION = '1.1_01' class PropDict: diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index cccd43ba3..d3aa594b4 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -13,7 +13,7 @@ platform API: that way platform APIs provided by the platform-specific support m are correctly composed into the base functionality. """ -API_VERSION = 5 +API_VERSION = '1.1_01' fdatasync = getattr(os, 'fdatasync', os.fsync) diff --git a/src/borg/platform/darwin.pyx b/src/borg/platform/darwin.pyx index 52b05a690..5800ac873 100644 --- a/src/borg/platform/darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -4,7 +4,7 @@ from ..helpers import user2uid, group2gid from ..helpers import safe_decode, safe_encode from .posix import swidth, umount -API_VERSION = 5 +API_VERSION = '1.1_01' cdef extern from "sys/acl.h": ctypedef struct _acl_t: diff --git a/src/borg/platform/freebsd.pyx b/src/borg/platform/freebsd.pyx index e63563db7..7a2763ce5 100644 --- a/src/borg/platform/freebsd.pyx +++ b/src/borg/platform/freebsd.pyx @@ -4,7 +4,7 @@ from ..helpers import posix_acl_use_stored_uid_gid from ..helpers import safe_encode, safe_decode from .posix import swidth, umount -API_VERSION = 5 +API_VERSION = '1.1_01' cdef extern from "errno.h": int errno diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index f96b6f892..e0ee8992f 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -13,7 +13,7 @@ from .posix import swidth from libc cimport errno from libc.stdint cimport int64_t -API_VERSION = 5 +API_VERSION = '1.1_01' cdef extern from "sys/types.h": int ACL_TYPE_ACCESS diff --git a/src/borg/remote.py b/src/borg/remote.py index 44b4a2482..2567e0ea1 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -11,6 +11,7 @@ import tempfile import time import traceback import textwrap +import time from subprocess import Popen, PIPE import msgpack @@ -39,6 +40,23 @@ MAX_INFLIGHT = 100 RATELIMIT_PERIOD = 0.1 +def os_write(fd, data): + """os.write wrapper so we do not lose data for partial writes.""" + # This is happening frequently on cygwin due to its small pipe buffer size of only 64kiB + # and also due to its different blocking pipe behaviour compared to Linux/*BSD. + # Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a + # signal, in which case serve() would terminate. + amount = remaining = len(data) + while remaining: + count = os.write(fd, data) + remaining -= count + if not remaining: + break + data = data[count:] + time.sleep(count * 1e-09) + return amount + + class ConnectionClosed(Error): """Connection closed by remote host""" @@ -176,7 +194,7 @@ class RepositoryServer: # pragma: no cover if self.repository is not None: self.repository.close() else: - os.write(stderr_fd, 'Borg {}: Got connection close before repository was opened.\n' + os_write(stderr_fd, 'Borg {}: Got connection close before repository was opened.\n' .format(__version__).encode()) return unpacker.feed(data) @@ -235,7 +253,7 @@ class RepositoryServer: # pragma: no cover b'exception_short': ex_short, b'sysinfo': sysinfo()}) - os.write(stdout_fd, msg) + os_write(stdout_fd, msg) else: if isinstance(e, (Repository.DoesNotExist, Repository.AlreadyExists, PathNotAllowed)): # These exceptions are reconstructed on the client end in RemoteRepository.call_many(), @@ -253,12 +271,12 @@ class RepositoryServer: # pragma: no cover logging.error(msg) logging.log(tb_log_level, tb) exc = 'Remote Exception (see remote log for the traceback)' - os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) + os_write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: if dictFormat: - os.write(stdout_fd, msgpack.packb({MSGID: msgid, RESULT: res})) + os_write(stdout_fd, msgpack.packb({MSGID: msgid, RESULT: res})) else: - os.write(stdout_fd, msgpack.packb((1, msgid, None, res))) + os_write(stdout_fd, msgpack.packb((1, msgid, None, res))) if es: self.repository.close() return