From 941b8d777802bbfecd8c700bd017b2add0fd3d27 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 13 Jan 2017 19:09:57 +0100 Subject: [PATCH 1/4] borg serve: fix transmission data loss of pipe writes, fixes #1268 This problem was found on cygwin/windows due to its small pipe buffer size of 64kiB. Due to that, bigger (like >64kiB) writes are always only partially done and os.write() returns the amount of data that was actually sent. the code previously did not use that return value and assumed that always all is sent, which led to a loss of the remainder of transmission data and usually some "unexpected RPC data format" error on the client side. Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a signal, in which case serve() would terminate. --- borg/remote.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index 0944997cb..30291cba3 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -7,6 +7,7 @@ import shlex import sys import tempfile import textwrap +import time from subprocess import Popen, PIPE from . import __version__ @@ -28,6 +29,23 @@ BUFSIZE = 10 * 1024 * 1024 MAX_INFLIGHT = 100 +def os_write(fd, data): + """os.write wrapper so we do not lose data for partial writes.""" + # This is happening frequently on cygwin due to its small pipe buffer size of only 64kiB + # and also due to its different blocking pipe behaviour compared to Linux/*BSD. + # Neither Linux nor *BSD ever do partial writes on blocking pipes, unless interrupted by a + # signal, in which case serve() would terminate. + amount = remaining = len(data) + while remaining: + count = os.write(fd, data) + remaining -= count + if not remaining: + break + data = data[count:] + time.sleep(count * 1e-09) + return amount + + class ConnectionClosed(Error): """Connection closed by remote host""" @@ -106,7 +124,7 @@ class RepositoryServer: # pragma: no cover if self.repository is not None: self.repository.close() else: - os.write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n" + os_write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n" .format(__version__).encode()) return unpacker.feed(data) @@ -133,9 +151,9 @@ class RepositoryServer: # pragma: no cover logging.exception('Borg %s: exception in RPC call:', __version__) logging.error(sysinfo()) exc = "Remote Exception (see remote log for the traceback)" - os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) + os_write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: - os.write(stdout_fd, msgpack.packb((1, msgid, None, res))) + os_write(stdout_fd, msgpack.packb((1, msgid, None, res))) if es: self.repository.close() return From 93d7d3c1dbc5e8ab67021b89c9f1b84fa46872b1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 13 Jan 2017 21:02:58 +0100 Subject: [PATCH 2/4] travis: require succeeding OS X tests, fixes #2028 --- .travis.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1433a173..e4c5fc7a4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,8 +36,6 @@ matrix: os: osx osx_image: xcode6.4 env: TOXENV=py36 - allow_failures: - - os: osx install: - ./.travis/install.sh From 1c854b9f600b93834cdde12ef0a6a60ca622bdcb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 13 Jan 2017 21:23:21 +0100 Subject: [PATCH 3/4] API_VERSION: use numberspaces, fixes #2023 like '_', e.g. '1.0_01' for version 01 (used in 1.0 maintenance branch). this avoids overlap and accidental collisions between different release branches. --- borg/chunker.pyx | 2 +- borg/crypto.pyx | 2 +- borg/hashindex.pyx | 2 +- borg/helpers.py | 8 ++++---- borg/platform.py | 2 +- borg/platform_darwin.pyx | 2 +- borg/platform_freebsd.pyx | 2 +- borg/platform_linux.pyx | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/borg/chunker.pyx b/borg/chunker.pyx index 0faa06f38..7ac664ed7 100644 --- a/borg/chunker.pyx +++ b/borg/chunker.pyx @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -API_VERSION = 2 +API_VERSION = '1.0_01' from libc.stdlib cimport free diff --git a/borg/crypto.pyx b/borg/crypto.pyx index f7beb3433..e4b3322fa 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -8,7 +8,7 @@ from math import ceil from libc.stdlib cimport malloc, free -API_VERSION = 3 +API_VERSION = '1.0_01' cdef extern from "openssl/rand.h": int RAND_bytes(unsigned char *buf, int num) diff --git a/borg/hashindex.pyx b/borg/hashindex.pyx index a27d0e8fc..1b0d07c2f 100644 --- a/borg/hashindex.pyx +++ b/borg/hashindex.pyx @@ -4,7 +4,7 @@ import os cimport cython from libc.stdint cimport uint32_t, UINT32_MAX, uint64_t -API_VERSION = 3 +API_VERSION = '1.0_01' cdef extern from "_hashindex.c": diff --git a/borg/helpers.py b/borg/helpers.py index 7c4a0fa91..4edb0313c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -88,13 +88,13 @@ class PlaceholderError(Error): def check_extension_modules(): from . import platform - if hashindex.API_VERSION != 3: + if hashindex.API_VERSION != '1.0_01': raise ExtensionModuleError - if chunker.API_VERSION != 2: + if chunker.API_VERSION != '1.0_01': raise ExtensionModuleError - if crypto.API_VERSION != 3: + if crypto.API_VERSION != '1.0_01': raise ExtensionModuleError - if platform.API_VERSION != 3: + if platform.API_VERSION != '1.0_01': raise ExtensionModuleError diff --git a/borg/platform.py b/borg/platform.py index be7a2bcd1..652ff8966 100644 --- a/borg/platform.py +++ b/borg/platform.py @@ -30,7 +30,7 @@ elif sys.platform.startswith('freebsd'): # pragma: freebsd only elif sys.platform == 'darwin': # pragma: darwin only from .platform_darwin import acl_get, acl_set, API_VERSION else: # pragma: unknown platform only - API_VERSION = 3 + API_VERSION = '1.0_01' def acl_get(path, item, st, numeric_owner=False): pass diff --git a/borg/platform_darwin.pyx b/borg/platform_darwin.pyx index 4dc25b83a..e421ce289 100644 --- a/borg/platform_darwin.pyx +++ b/borg/platform_darwin.pyx @@ -1,7 +1,7 @@ import os from .helpers import user2uid, group2gid, safe_decode, safe_encode -API_VERSION = 3 +API_VERSION = '1.0_01' cdef extern from "sys/acl.h": ctypedef struct _acl_t: diff --git a/borg/platform_freebsd.pyx b/borg/platform_freebsd.pyx index ae69af68a..27796c318 100644 --- a/borg/platform_freebsd.pyx +++ b/borg/platform_freebsd.pyx @@ -1,7 +1,7 @@ import os from .helpers import posix_acl_use_stored_uid_gid, safe_encode, safe_decode -API_VERSION = 3 +API_VERSION = '1.0_01' cdef extern from "errno.h": int errno diff --git a/borg/platform_linux.pyx b/borg/platform_linux.pyx index 0185268c1..5f1e8aef2 100644 --- a/borg/platform_linux.pyx +++ b/borg/platform_linux.pyx @@ -4,7 +4,7 @@ import subprocess from stat import S_ISLNK from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode -API_VERSION = 3 +API_VERSION = '1.0_01' cdef extern from "sys/types.h": int ACL_TYPE_ACCESS From 022c1288e7a5ea983729eab92c84bab593e95edb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 13 Jan 2017 21:49:06 +0100 Subject: [PATCH 4/4] borg create: document how to backup stdin, fixes #2013 --- borg/archiver.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 032d600c6..667a8e90b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1311,8 +1311,12 @@ class Archiver: create_epilog = textwrap.dedent(""" This command creates a backup archive containing all files found while recursively - traversing all paths specified. The archive will consume almost no disk space for - files or parts of files that have already been stored in other archives. + traversing all paths specified. When giving '-' as path, borg will read data + from standard input and create a file 'stdin' in the created archive from that + data. + + The archive will consume almost no disk space for files or parts of files that + have already been stored in other archives. The archive name needs to be unique. It must not end in '.checkpoint' or '.checkpoint.N' (with N being a number), because these names are used for