diff --git a/.travis.yml b/.travis.yml index 9a853c40..3b62d1eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ matrix: os: linux dist: trusty env: TOXENV=py35 - - python: 3.6-dev + - python: 3.6 os: linux dist: trusty env: TOXENV=py36 @@ -33,6 +33,10 @@ matrix: os: osx osx_image: xcode6.4 env: TOXENV=py35 + - language: generic + os: osx + osx_image: xcode6.4 + env: TOXENV=py36 allow_failures: - os: osx diff --git a/.travis/install.sh b/.travis/install.sh index 309a3e9d..1147f8ef 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -18,7 +18,7 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then brew install xz # required for python lzma module brew outdated pyenv || brew upgrade pyenv brew install pkg-config - brew install Caskroom/versions/osxfuse + brew install Caskroom/cask/osxfuse case "${TOXENV}" in py34) @@ -29,6 +29,10 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then pyenv install 3.5.1 pyenv global 3.5.1 ;; + py36) + pyenv install 3.6.0 + pyenv global 3.6.0 + ;; esac pyenv rehash python -m pip install --user virtualenv diff --git a/Vagrantfile b/Vagrantfile index 8aa3aef0..1fe981f1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -224,6 +224,7 @@ def install_pythons(boxname) . ~/.bash_profile pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests + pyenv install 3.6.0 # tests pyenv install 3.5.2 # binary build, use latest 3.5.x release pyenv rehash EOF @@ -317,8 +318,8 @@ def run_tests(boxname) . ../borg-env/bin/activate if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.4.0 3.5.0 - pyenv local 3.4.0 3.5.0 + pyenv global 3.4.0 3.5.0 3.6.0 + pyenv local 3.4.0 3.5.0 3.6.0 fi # otherwise: just use the system python if which fakeroot 2> /dev/null; then diff --git a/docs/changes.rst b/docs/changes.rst index ba951d17..ba4dd4e9 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -204,6 +204,43 @@ Other changes: - point XDG_*_HOME to temp dirs for tests, #1714 - remove all BORG_* env vars from the outer environment + +Version 1.0.10rc1 (not released yet) +------------------------------------ + +Bug fixes: + +- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992 +- When running out of buffer memory when reading xattrs, only skip the + current file, #1993 +- Fixed "borg upgrade --tam" crashing with unencrypted repositories. Since + :ref:`the issue ` is not relevant for unencrypted repositories, + it now does nothing and prints an error, #1981. +- Fixed change-passphrase crashing with unencrypted repositories, #1978 +- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997 +- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled + +Other changes: + +- xattr: ignore empty names returned by llistxattr(2) et al +- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT, + SIGBUS and SIGILL signals to dump the Python traceback. +- Also print a traceback on SIGUSR2. +- borg change-passphrase: print key location (simplify making a backup of it) +- officially support Python 3.6 (setup.py: add Python 3.6 qualifier) +- tests: + + - vagrant / travis / tox: add Python 3.6 based testing + - travis: fix osxfuse install (fixes OS X testing on Travis CI) + - use pytest-xdist to parallelize testing +- docs: + + - language clarification - VM backup FAQ +- fix typos (taken from Debian package patch) +- remote: include data hexdump in "unexpected RPC data" error message +- remote: log SSH command line at debug level + + Version 1.0.9 (2016-12-20) -------------------------- diff --git a/docs/faq.rst b/docs/faq.rst index cad34e20..c41b1a95 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -13,7 +13,7 @@ Yes, the `deduplication`_ technique used by Also, we have optional simple sparse file support for extract. If you use non-snapshotting backup tools like Borg to back up virtual machines, -then these should be turned off for doing so. Backing up live VMs this way can (and will) +then the VMs should be turned off for the duration of the backup. Backing up live VMs can (and will) result in corrupted or inconsistent backup contents: a VM image is just a regular file to Borg with the same issues as regular files when it comes to concurrent reading and writing from the same file. diff --git a/docs/usage.rst b/docs/usage.rst index 8410d115..85b5a7cd 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -164,7 +164,7 @@ General: BORG_FILES_CACHE_TTL When set to a numeric value, this determines the maximum "time to live" for the files cache entries (default: 20). The files cache is used to quickly determine whether a file is unchanged. - The FAQ explains this more detailled in: :ref:`always_chunking` + The FAQ explains this more detailed in: :ref:`always_chunking` TMPDIR where temporary files are stored (might need a lot of temporary space for some operations) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index f14c3abf..e3417060 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -1,6 +1,7 @@ virtualenv tox pytest +pytest-xdist pytest-cov pytest-benchmark Cython diff --git a/setup.py b/setup.py index 26367ef1..902157b3 100644 --- a/setup.py +++ b/setup.py @@ -409,6 +409,7 @@ setup( 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', 'Topic :: Security :: Cryptography', 'Topic :: System :: Archiving :: Backup', ], diff --git a/src/borg/archive.py b/src/borg/archive.py index 90759dae..25983a3c 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1360,16 +1360,18 @@ class ArchiveChecker: sort_by = sort_by.split(',') if any((first, last, prefix)): archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last) + if not archive_infos: + logger.warning('--first/--last/--prefix did not match any archives') else: archive_infos = self.manifest.archives.list(sort_by=sort_by) else: # we only want one specific archive - info = self.manifest.archives.get(archive) - if info is None: + try: + archive_infos = [self.manifest.archives[archive]] + except KeyError: logger.error("Archive '%s' not found.", archive) - archive_infos = [] - else: - archive_infos = [info] + self.error_found = True + return num_archives = len(archive_infos) with cache_if_remote(self.repository) as repository: diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 777130b4..c7b3b70b 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1,5 +1,6 @@ import argparse import collections +import faulthandler import functools import hashlib import inspect @@ -240,8 +241,14 @@ class Archiver: @with_repository() def do_change_passphrase(self, args, repository, manifest, key): """Change repository key file passphrase""" + if not hasattr(key, 'change_passphrase'): + print('This repository is not encrypted, cannot change the passphrase.') + return EXIT_ERROR key.change_passphrase() logger.info('Key updated') + if hasattr(key, 'find_key'): + # print key location to make backing it up easier + logger.info('Key location: %s', key.find_key()) return EXIT_SUCCESS @with_repository(lock=False, exclusive=False, manifest=False, cache=False) @@ -1078,6 +1085,10 @@ class Archiver: if args.tam: manifest, key = Manifest.load(repository, force_tam_not_required=args.force) + if not hasattr(key, 'change_passphrase'): + print('This repository is not encrypted, cannot enable TAM.') + return EXIT_ERROR + if not manifest.tam_verified or not manifest.config.get(b'tam_required', False): # The standard archive listing doesn't include the archive ID like in borg 1.1.x print('Manifest contents:') @@ -2390,7 +2401,7 @@ class Archiver: Upgrade an existing Borg repository. Borg 1.x.y upgrades - ------------------- + +++++++++++++++++++ Use ``borg upgrade --tam REPO`` to require manifest authentication introduced with Borg 1.0.9 to address security issues. This means @@ -2412,7 +2423,7 @@ class Archiver: for details. Attic and Borg 0.xx to Borg 1.x - ------------------------------- + +++++++++++++++++++++++++++++++ This currently supports converting an Attic repository to Borg and also helps with converting Borg 0.xx to 1.0. @@ -2852,6 +2863,11 @@ def sig_info_handler(sig_no, stack): # pragma: no cover break +def sig_trace_handler(sig_no, stack): # pragma: no cover + print('\nReceived SIGUSR2 at %s, dumping trace...' % datetime.now().replace(microsecond=0), file=sys.stderr) + faulthandler.dump_traceback() + + def main(): # pragma: no cover # provide 'borg mount' behaviour when the main script/executable is named borgfs if os.path.basename(sys.argv[0]) == "borgfs": @@ -2868,10 +2884,14 @@ def main(): # pragma: no cover # SIGHUP is important especially for systemd systems, where logind # sends it when a session exits, in addition to any traditional use. # Output some info if we receive SIGUSR1 or SIGINFO (ctrl-t). + + # Register fault handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL. + faulthandler.enable() with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \ signal_handler('SIGHUP', raising_signal_handler(SigHup)), \ signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \ signal_handler('SIGUSR1', sig_info_handler), \ + signal_handler('SIGUSR2', sig_trace_handler), \ signal_handler('SIGINFO', sig_info_handler): archiver = Archiver() msg = tb = None diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 9b10a984..7c71cc91 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -60,9 +60,15 @@ class Error(Exception): # show a traceback? traceback = False + def __init__(self, *args): + super().__init__(*args) + self.args = args + def get_message(self): return type(self).__doc__.format(*self.args) + __str__ = get_message + class ErrorWithTraceback(Error): """like Error, but show a traceback also""" @@ -798,6 +804,10 @@ class Buffer: """ provide a thread-local buffer """ + + class MemoryLimitExceeded(Error, OSError): + """Requested buffer size {} is above the limit of {}.""" + def __init__(self, allocator, size=4096, limit=None): """ Initialize the buffer: use allocator(size) call to allocate a buffer. @@ -817,11 +827,11 @@ class Buffer: """ resize the buffer - to avoid frequent reallocation, we usually always grow (if needed). giving init=True it is possible to first-time initialize or shrink the buffer. - if a buffer size beyond the limit is requested, raise ValueError. + if a buffer size beyond the limit is requested, raise Buffer.MemoryLimitExceeded (OSError). """ size = int(size) if self.limit is not None and size > self.limit: - raise ValueError('Requested buffer size %d is above the limit of %d.' % (size, self.limit)) + raise Buffer.MemoryLimitExceeded(size, self.limit) if init or len(self) < size: self._thread_local.buffer = self.allocator(size) diff --git a/src/borg/remote.py b/src/borg/remote.py index 9dc85f1e..44b4a248 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -10,6 +10,7 @@ import sys import tempfile import time import traceback +import textwrap from subprocess import Popen, PIPE import msgpack @@ -23,6 +24,9 @@ from .helpers import replace_placeholders from .helpers import yes from .repository import Repository from .version import parse_version, format_version +from .logger import create_logger + +logger = create_logger(__name__) RPC_PROTOCOL_VERSION = 2 BORG_VERSION = parse_version(__version__) @@ -56,7 +60,16 @@ class UnexpectedRPCDataFormatFromClient(Error): class UnexpectedRPCDataFormatFromServer(Error): - """Got unexpected RPC data format from server.""" + """Got unexpected RPC data format from server:\n{}""" + + def __init__(self, data): + try: + data = data.decode()[:128] + except UnicodeDecodeError: + data = data[:128] + data = ['%02X' % byte for byte in data] + data = textwrap.fill(' '.join(data), 16 * 3) + super().__init__(data) # Protocol compatibility: @@ -476,6 +489,7 @@ class RemoteRepository: env.pop(lp_key, None) env.pop('BORG_PASSPHRASE', None) # security: do not give secrets to subprocess env['BORG_VERSION'] = __version__ + logger.debug('SSH command line: %s', borg_cmd) self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() @@ -685,7 +699,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+. else: unpacked = {MSGID: msgid, RESULT: res} else: - raise UnexpectedRPCDataFormatFromServer() + raise UnexpectedRPCDataFormatFromServer(data) if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) if b'exception_class' in unpacked: diff --git a/src/borg/testsuite/archive.py b/src/borg/testsuite/archive.py index 3f315fe4..a4f613d1 100644 --- a/src/borg/testsuite/archive.py +++ b/src/borg/testsuite/archive.py @@ -1,4 +1,5 @@ import os +from collections import OrderedDict from datetime import datetime, timezone from io import StringIO from unittest.mock import Mock @@ -201,11 +202,15 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized): assert not valid_msgpacked_dict(packed, item_keys_serialized) +# pytest-xdist requires always same order for the keys and dicts: +IK = sorted(list(ITEM_KEYS)) + + @pytest.mark.parametrize('packed', [msgpack.packb(o) for o in [ {b'path': b'/a/b/c'}, # small (different msgpack mapping type!) - dict((k, b'') for k in ITEM_KEYS), # as big (key count) as it gets - dict((k, b'x' * 1000) for k in ITEM_KEYS), # as big (key count and volume) as it gets + OrderedDict((k, b'') for k in IK), # as big (key count) as it gets + OrderedDict((k, b'x' * 1000) for k in IK), # as big (key count and volume) as it gets ]]) def test_valid_msgpacked_items(packed, item_keys_serialized): assert valid_msgpacked_dict(packed, item_keys_serialized) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 2d97ce06..6da3073a 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2358,6 +2358,82 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase): assert not self.cmd('list', self.repository_location) +class ManifestAuthenticationTest(ArchiverTestCaseBase): + def spoof_manifest(self, repository): + with repository: + _, key = Manifest.load(repository) + repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ + 'version': 1, + 'archives': {}, + 'config': {}, + 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + }))) + repository.commit() + + def test_fresh_init_tam_required(self): + self.cmd('init', self.repository_location) + repository = Repository(self.repository_path, exclusive=True) + with repository: + manifest, key = Manifest.load(repository) + repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ + 'version': 1, + 'archives': {}, + 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + }))) + repository.commit() + + with pytest.raises(TAMRequiredError): + self.cmd('list', self.repository_location) + + def test_not_required(self): + self.cmd('init', self.repository_location) + self.create_src_archive('archive1234') + repository = Repository(self.repository_path, exclusive=True) + with repository: + shutil.rmtree(get_security_dir(bin_to_hex(repository.id))) + _, key = Manifest.load(repository) + key.tam_required = False + key.change_passphrase(key._passphrase) + + manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID))) + del manifest[b'tam'] + repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest))) + repository.commit() + output = self.cmd('list', '--debug', self.repository_location) + assert 'archive1234' in output + assert 'TAM not found and not required' in output + # Run upgrade + self.cmd('upgrade', '--tam', self.repository_location) + # Manifest must be authenticated now + output = self.cmd('list', '--debug', self.repository_location) + assert 'archive1234' in output + assert 'TAM-verified manifest' in output + # Try to spoof / modify pre-1.0.9 + self.spoof_manifest(repository) + # Fails + with pytest.raises(TAMRequiredError): + self.cmd('list', self.repository_location) + # Force upgrade + self.cmd('upgrade', '--tam', '--force', self.repository_location) + self.cmd('list', self.repository_location) + + def test_disable(self): + self.cmd('init', self.repository_location) + self.create_src_archive('archive1234') + self.cmd('upgrade', '--disable-tam', self.repository_location) + repository = Repository(self.repository_path, exclusive=True) + self.spoof_manifest(repository) + assert not self.cmd('list', self.repository_location) + + def test_disable2(self): + self.cmd('init', self.repository_location) + self.create_src_archive('archive1234') + repository = Repository(self.repository_path, exclusive=True) + self.spoof_manifest(repository) + self.cmd('upgrade', '--disable-tam', self.repository_location) + assert not self.cmd('list', self.repository_location) + + @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs') class RemoteArchiverTestCase(ArchiverTestCase): prefix = '__testsuite__:' diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 6277d98d..ed2b0157 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -783,7 +783,7 @@ class TestBuffer: buffer = Buffer(bytearray, size=100, limit=200) buffer.resize(200) assert len(buffer) == 200 - with pytest.raises(ValueError): + with pytest.raises(Buffer.MemoryLimitExceeded): buffer.resize(201) assert len(buffer) == 200 @@ -797,7 +797,7 @@ class TestBuffer: b3 = buffer.get(200) assert len(b3) == 200 assert b3 is not b2 # new, resized buffer - with pytest.raises(ValueError): + with pytest.raises(Buffer.MemoryLimitExceeded): buffer.get(201) # beyond limit assert len(buffer) == 200 diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 8d77e120..2819c64c 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -717,6 +717,8 @@ class RemoteRepositoryTestCase(RepositoryTestCase): assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve'] args = MockArgs() + # XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown: + logging.getLogger().setLevel(logging.INFO) # note: test logger is on info log level, so --info gets added automagically assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info'] args.remote_path = 'borg-0.28.2' diff --git a/src/borg/xattr.py b/src/borg/xattr.py index c3d51cb2..c37d1a81 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -111,7 +111,7 @@ def split_lstring(buf): class BufferTooSmallError(Exception): - """the buffer given to an xattr function was too small for the result""" + """the buffer given to an xattr function was too small for the result.""" def _check(rv, path=None, detect_buffer_too_small=False): @@ -202,7 +202,7 @@ if sys.platform.startswith('linux'): # pragma: linux only n, buf = _listxattr_inner(func, path) return [os.fsdecode(name) for name in split_string0(buf[:n]) - if not name.startswith(b'system.posix_acl_')] + if name and not name.startswith(b'system.posix_acl_')] def getxattr(path, name, *, follow_symlinks=True): def func(path, name, buf, size): @@ -258,7 +258,7 @@ elif sys.platform == 'darwin': # pragma: darwin only return libc.listxattr(path, buf, size, XATTR_NOFOLLOW) n, buf = _listxattr_inner(func, path) - return [os.fsdecode(name) for name in split_string0(buf[:n])] + return [os.fsdecode(name) for name in split_string0(buf[:n]) if name] def getxattr(path, name, *, follow_symlinks=True): def func(path, name, buf, size): @@ -317,7 +317,7 @@ elif sys.platform.startswith('freebsd'): # pragma: freebsd only return libc.extattr_list_link(path, ns, buf, size) n, buf = _listxattr_inner(func, path) - return [os.fsdecode(name) for name in split_lstring(buf[:n])] + return [os.fsdecode(name) for name in split_lstring(buf[:n]) if name] def getxattr(path, name, *, follow_symlinks=True): def func(path, name, buf, size): diff --git a/tox.ini b/tox.ini index 5aa14a50..9c7cea25 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = -rrequirements.d/development.txt -rrequirements.d/attic.txt -rrequirements.d/fuse.txt -commands = py.test -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} +commands = py.test -n 8 -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = *