mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge branch '1.0-maint' into merge/1.0-maint 

# Conflicts: ... everywhere ...
#	.travis.yml
#	Vagrantfile
#	borg/testsuite/key.py
#	docs/changes.rst
#	docs/quickstart.rst
#	docs/usage.rst
#	docs/usage/upgrade.rst.inc
#	src/borg/archive.py
#	src/borg/archiver.py
#	src/borg/crypto.pyx
#	src/borg/helpers.py
#	src/borg/key.py
#	src/borg/remote.py
#	src/borg/repository.py
#	src/borg/testsuite/archive.py
#	src/borg/testsuite/archiver.py
#	src/borg/testsuite/crypto.py
#	src/borg/testsuite/helpers.py
#	src/borg/testsuite/repository.py
#	src/borg/upgrader.py
#	tox.ini
This commit is contained in:
Marian Beermann 2017-01-12 15:01:41 +01:00
parent 745a3a5753 9c42a75831
commit ecad0ed53a
18 changed files with 203 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.travis.yml
View File

@ -17,7 +17,7 @@ matrix:
os: linux
dist: trusty
env: TOXENV=py35
- python: 3.6-dev
- python: 3.6
os: linux
dist: trusty
env: TOXENV=py36
@ -33,6 +33,10 @@ matrix:
os: osx
osx_image: xcode6.4
env: TOXENV=py35
- language: generic
os: osx
osx_image: xcode6.4
env: TOXENV=py36
allow_failures:
- os: osx

6
.travis/install.sh
View File

@ -18,7 +18,7 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
brew install xz # required for python lzma module
brew outdated pyenv || brew upgrade pyenv
brew install pkg-config
brew install Caskroom/versions/osxfuse
brew install Caskroom/cask/osxfuse
case "${TOXENV}" in
py34)
@ -29,6 +29,10 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then
pyenv install 3.5.1
pyenv global 3.5.1
;;
py36)
pyenv install 3.6.0
pyenv global 3.6.0
;;
esac
pyenv rehash
python -m pip install --user virtualenv

5
Vagrantfile vendored
View File

@ -224,6 +224,7 @@ def install_pythons(boxname)
. ~/.bash_profile
pyenv install 3.4.0 # tests
pyenv install 3.5.0 # tests
pyenv install 3.6.0 # tests
pyenv install 3.5.2 # binary build, use latest 3.5.x release
pyenv rehash
EOF
@ -317,8 +318,8 @@ def run_tests(boxname)
. ../borg-env/bin/activate
if which pyenv 2> /dev/null; then
# for testing, use the earliest point releases of the supported python versions:
pyenv global 3.4.0 3.5.0
pyenv local 3.4.0 3.5.0
pyenv global 3.4.0 3.5.0 3.6.0
pyenv local 3.4.0 3.5.0 3.6.0
fi
# otherwise: just use the system python
if which fakeroot 2> /dev/null; then

37
docs/changes.rst
View File

@ -204,6 +204,43 @@ Other changes:
- point XDG_*_HOME to temp dirs for tests, #1714
- remove all BORG_* env vars from the outer environment
Version 1.0.10rc1 (not released yet)
------------------------------------
Bug fixes:
- Avoid triggering an ObjectiveFS bug in xattr retrieval, #1992
- When running out of buffer memory when reading xattrs, only skip the
current file, #1993
- Fixed "borg upgrade --tam" crashing with unencrypted repositories. Since
:ref:`the issue <tam_vuln>` is not relevant for unencrypted repositories,
it now does nothing and prints an error, #1981.
- Fixed change-passphrase crashing with unencrypted repositories, #1978
- Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997
- borg check: print non-exit-code warning if --last or --prefix aren't fulfilled
Other changes:
- xattr: ignore empty names returned by llistxattr(2) et al
- Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT,
SIGBUS and SIGILL signals to dump the Python traceback.
- Also print a traceback on SIGUSR2.
- borg change-passphrase: print key location (simplify making a backup of it)
- officially support Python 3.6 (setup.py: add Python 3.6 qualifier)
- tests:
- vagrant / travis / tox: add Python 3.6 based testing
- travis: fix osxfuse install (fixes OS X testing on Travis CI)
- use pytest-xdist to parallelize testing
- docs:
- language clarification - VM backup FAQ
- fix typos (taken from Debian package patch)
- remote: include data hexdump in "unexpected RPC data" error message
- remote: log SSH command line at debug level
Version 1.0.9 (2016-12-20)
--------------------------

2
docs/faq.rst
View File

@ -13,7 +13,7 @@ Yes, the `deduplication`_ technique used by
Also, we have optional simple sparse file support for extract.
If you use non-snapshotting backup tools like Borg to back up virtual machines,
then these should be turned off for doing so. Backing up live VMs this way can (and will)
then the VMs should be turned off for the duration of the backup. Backing up live VMs can (and will)
result in corrupted or inconsistent backup contents: a VM image is just a regular file to
Borg with the same issues as regular files when it comes to concurrent reading and writing from
the same file.

2
docs/usage.rst
View File

@ -164,7 +164,7 @@ General:
BORG_FILES_CACHE_TTL
When set to a numeric value, this determines the maximum "time to live" for the files cache
entries (default: 20). The files cache is used to quickly determine whether a file is unchanged.
The FAQ explains this more detailled in: :ref:`always_chunking`
The FAQ explains this more detailed in: :ref:`always_chunking`
TMPDIR
where temporary files are stored (might need a lot of temporary space for some operations)

1
requirements.d/development.txt
View File

@ -1,6 +1,7 @@
virtualenv
tox
pytest
pytest-xdist
pytest-cov
pytest-benchmark
Cython

1
setup.py
View File

@ -409,6 +409,7 @@ setup(
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Topic :: Security :: Cryptography',
'Topic :: System :: Archiving :: Backup',
],

12
src/borg/archive.py
View File

@ -1360,16 +1360,18 @@ class ArchiveChecker:
sort_by = sort_by.split(',')
if any((first, last, prefix)):
archive_infos = self.manifest.archives.list(sort_by=sort_by, prefix=prefix, first=first, last=last)
if not archive_infos:
logger.warning('--first/--last/--prefix did not match any archives')
else:
archive_infos = self.manifest.archives.list(sort_by=sort_by)
else:
# we only want one specific archive
info = self.manifest.archives.get(archive)
if info is None:
try:
archive_infos = [self.manifest.archives[archive]]
except KeyError:
logger.error("Archive '%s' not found.", archive)
archive_infos = []
else:
archive_infos = [info]
self.error_found = True
return
num_archives = len(archive_infos)
with cache_if_remote(self.repository) as repository:

24
src/borg/archiver.py
View File

@ -1,5 +1,6 @@
import argparse
import collections
import faulthandler
import functools
import hashlib
import inspect
@ -240,8 +241,14 @@ class Archiver:
@with_repository()
def do_change_passphrase(self, args, repository, manifest, key):
"""Change repository key file passphrase"""
if not hasattr(key, 'change_passphrase'):
print('This repository is not encrypted, cannot change the passphrase.')
return EXIT_ERROR
key.change_passphrase()
logger.info('Key updated')
if hasattr(key, 'find_key'):
# print key location to make backing it up easier
logger.info('Key location: %s', key.find_key())
return EXIT_SUCCESS
@with_repository(lock=False, exclusive=False, manifest=False, cache=False)
@ -1078,6 +1085,10 @@ class Archiver:
if args.tam:
manifest, key = Manifest.load(repository, force_tam_not_required=args.force)
if not hasattr(key, 'change_passphrase'):
print('This repository is not encrypted, cannot enable TAM.')
return EXIT_ERROR
if not manifest.tam_verified or not manifest.config.get(b'tam_required', False):
# The standard archive listing doesn't include the archive ID like in borg 1.1.x
print('Manifest contents:')
@ -2390,7 +2401,7 @@ class Archiver:
Upgrade an existing Borg repository.
Borg 1.x.y upgrades
-------------------
+++++++++++++++++++
Use ``borg upgrade --tam REPO`` to require manifest authentication
introduced with Borg 1.0.9 to address security issues. This means
@ -2412,7 +2423,7 @@ class Archiver:
for details.
Attic and Borg 0.xx to Borg 1.x
-------------------------------
+++++++++++++++++++++++++++++++
This currently supports converting an Attic repository to Borg and also
helps with converting Borg 0.xx to 1.0.
@ -2852,6 +2863,11 @@ def sig_info_handler(sig_no, stack): # pragma: no cover
break
def sig_trace_handler(sig_no, stack): # pragma: no cover
print('\nReceived SIGUSR2 at %s, dumping trace...' % datetime.now().replace(microsecond=0), file=sys.stderr)
faulthandler.dump_traceback()
def main(): # pragma: no cover
# provide 'borg mount' behaviour when the main script/executable is named borgfs
if os.path.basename(sys.argv[0]) == "borgfs":
@ -2868,10 +2884,14 @@ def main(): # pragma: no cover
# SIGHUP is important especially for systemd systems, where logind
# sends it when a session exits, in addition to any traditional use.
# Output some info if we receive SIGUSR1 or SIGINFO (ctrl-t).
# Register fault handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL.
faulthandler.enable()
with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \
signal_handler('SIGHUP', raising_signal_handler(SigHup)), \
signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \
signal_handler('SIGUSR1', sig_info_handler), \
signal_handler('SIGUSR2', sig_trace_handler), \
signal_handler('SIGINFO', sig_info_handler):
archiver = Archiver()
msg = tb = None

14
src/borg/helpers.py
View File

@ -60,9 +60,15 @@ class Error(Exception):
# show a traceback?
traceback = False
def __init__(self, *args):
super().__init__(*args)
self.args = args
def get_message(self):
return type(self).__doc__.format(*self.args)
__str__ = get_message
class ErrorWithTraceback(Error):
"""like Error, but show a traceback also"""
@ -798,6 +804,10 @@ class Buffer:
"""
provide a thread-local buffer
"""
class MemoryLimitExceeded(Error, OSError):
"""Requested buffer size {} is above the limit of {}."""
def __init__(self, allocator, size=4096, limit=None):
"""
Initialize the buffer: use allocator(size) call to allocate a buffer.
@ -817,11 +827,11 @@ class Buffer:
"""
resize the buffer - to avoid frequent reallocation, we usually always grow (if needed).
giving init=True it is possible to first-time initialize or shrink the buffer.
if a buffer size beyond the limit is requested, raise ValueError.
if a buffer size beyond the limit is requested, raise Buffer.MemoryLimitExceeded (OSError).
"""
size = int(size)
if self.limit is not None and size > self.limit:
raise ValueError('Requested buffer size %d is above the limit of %d.' % (size, self.limit))
raise Buffer.MemoryLimitExceeded(size, self.limit)
if init or len(self) < size:
self._thread_local.buffer = self.allocator(size)

18
src/borg/remote.py
View File

@ -10,6 +10,7 @@ import sys
import tempfile
import time
import traceback
import textwrap
from subprocess import Popen, PIPE
import msgpack
@ -23,6 +24,9 @@ from .helpers import replace_placeholders
from .helpers import yes
from .repository import Repository
from .version import parse_version, format_version
from .logger import create_logger
logger = create_logger(__name__)
RPC_PROTOCOL_VERSION = 2
BORG_VERSION = parse_version(__version__)
@ -56,7 +60,16 @@ class UnexpectedRPCDataFormatFromClient(Error):
class UnexpectedRPCDataFormatFromServer(Error):
"""Got unexpected RPC data format from server."""
"""Got unexpected RPC data format from server:\n{}"""
def __init__(self, data):
try:
data = data.decode()[:128]
except UnicodeDecodeError:
data = data[:128]
data = ['%02X' % byte for byte in data]
data = textwrap.fill(' '.join(data), 16 * 3)
super().__init__(data)
# Protocol compatibility:
@ -476,6 +489,7 @@ class RemoteRepository:
env.pop(lp_key, None)
env.pop('BORG_PASSPHRASE', None) # security: do not give secrets to subprocess
env['BORG_VERSION'] = __version__
logger.debug('SSH command line: %s', borg_cmd)
self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
self.stdin_fd = self.p.stdin.fileno()
self.stdout_fd = self.p.stdout.fileno()
@ -685,7 +699,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
else:
unpacked = {MSGID: msgid, RESULT: res}
else:
raise UnexpectedRPCDataFormatFromServer()
raise UnexpectedRPCDataFormatFromServer(data)
if msgid in self.ignore_responses:
self.ignore_responses.remove(msgid)
if b'exception_class' in unpacked:

9
src/borg/testsuite/archive.py
View File

@ -1,4 +1,5 @@
import os
from collections import OrderedDict
from datetime import datetime, timezone
from io import StringIO
from unittest.mock import Mock
@ -201,11 +202,15 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized):
assert not valid_msgpacked_dict(packed, item_keys_serialized)
# pytest-xdist requires always same order for the keys and dicts:
IK = sorted(list(ITEM_KEYS))
@pytest.mark.parametrize('packed',
[msgpack.packb(o) for o in [
{b'path': b'/a/b/c'}, # small (different msgpack mapping type!)
dict((k, b'') for k in ITEM_KEYS), # as big (key count) as it gets
dict((k, b'x' * 1000) for k in ITEM_KEYS), # as big (key count and volume) as it gets
OrderedDict((k, b'') for k in IK), # as big (key count) as it gets
OrderedDict((k, b'x' * 1000) for k in IK), # as big (key count and volume) as it gets
]])
def test_valid_msgpacked_items(packed, item_keys_serialized):
assert valid_msgpacked_dict(packed, item_keys_serialized)

76
src/borg/testsuite/archiver.py
View File

@ -2358,6 +2358,82 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
assert not self.cmd('list', self.repository_location)
class ManifestAuthenticationTest(ArchiverTestCaseBase):
def spoof_manifest(self, repository):
with repository:
_, key = Manifest.load(repository)
repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({
'version': 1,
'archives': {},
'config': {},
'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
})))
repository.commit()
def test_fresh_init_tam_required(self):
self.cmd('init', self.repository_location)
repository = Repository(self.repository_path, exclusive=True)
with repository:
manifest, key = Manifest.load(repository)
repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({
'version': 1,
'archives': {},
'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(),
})))
repository.commit()
with pytest.raises(TAMRequiredError):
self.cmd('list', self.repository_location)
def test_not_required(self):
self.cmd('init', self.repository_location)
self.create_src_archive('archive1234')
repository = Repository(self.repository_path, exclusive=True)
with repository:
shutil.rmtree(get_security_dir(bin_to_hex(repository.id)))
_, key = Manifest.load(repository)
key.tam_required = False
key.change_passphrase(key._passphrase)
manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID)))
del manifest[b'tam']
repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest)))
repository.commit()
output = self.cmd('list', '--debug', self.repository_location)
assert 'archive1234' in output
assert 'TAM not found and not required' in output
# Run upgrade
self.cmd('upgrade', '--tam', self.repository_location)
# Manifest must be authenticated now
output = self.cmd('list', '--debug', self.repository_location)
assert 'archive1234' in output
assert 'TAM-verified manifest' in output
# Try to spoof / modify pre-1.0.9
self.spoof_manifest(repository)
# Fails
with pytest.raises(TAMRequiredError):
self.cmd('list', self.repository_location)
# Force upgrade
self.cmd('upgrade', '--tam', '--force', self.repository_location)
self.cmd('list', self.repository_location)
def test_disable(self):
self.cmd('init', self.repository_location)
self.create_src_archive('archive1234')
self.cmd('upgrade', '--disable-tam', self.repository_location)
repository = Repository(self.repository_path, exclusive=True)
self.spoof_manifest(repository)
assert not self.cmd('list', self.repository_location)
def test_disable2(self):
self.cmd('init', self.repository_location)
self.create_src_archive('archive1234')
repository = Repository(self.repository_path, exclusive=True)
self.spoof_manifest(repository)
self.cmd('upgrade', '--disable-tam', self.repository_location)
assert not self.cmd('list', self.repository_location)
@pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs')
class RemoteArchiverTestCase(ArchiverTestCase):
prefix = '__testsuite__:'

4
src/borg/testsuite/helpers.py
View File

@ -783,7 +783,7 @@ class TestBuffer:
buffer = Buffer(bytearray, size=100, limit=200)
buffer.resize(200)
assert len(buffer) == 200
with pytest.raises(ValueError):
with pytest.raises(Buffer.MemoryLimitExceeded):
buffer.resize(201)
assert len(buffer) == 200
@ -797,7 +797,7 @@ class TestBuffer:
b3 = buffer.get(200)
assert len(b3) == 200
assert b3 is not b2 # new, resized buffer
with pytest.raises(ValueError):
with pytest.raises(Buffer.MemoryLimitExceeded):
buffer.get(201) # beyond limit
assert len(buffer) == 200

2
src/borg/testsuite/repository.py
View File

@ -717,6 +717,8 @@ class RemoteRepositoryTestCase(RepositoryTestCase):
assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve']
args = MockArgs()
# XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
logging.getLogger().setLevel(logging.INFO)
# note: test logger is on info log level, so --info gets added automagically
assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info']
args.remote_path = 'borg-0.28.2'

8
src/borg/xattr.py
View File

@ -111,7 +111,7 @@ def split_lstring(buf):
class BufferTooSmallError(Exception):
"""the buffer given to an xattr function was too small for the result"""
"""the buffer given to an xattr function was too small for the result."""
def _check(rv, path=None, detect_buffer_too_small=False):
@ -202,7 +202,7 @@ if sys.platform.startswith('linux'): # pragma: linux only
n, buf = _listxattr_inner(func, path)
return [os.fsdecode(name) for name in split_string0(buf[:n])
if not name.startswith(b'system.posix_acl_')]
if name and not name.startswith(b'system.posix_acl_')]
def getxattr(path, name, *, follow_symlinks=True):
def func(path, name, buf, size):
@ -258,7 +258,7 @@ elif sys.platform == 'darwin': # pragma: darwin only
return libc.listxattr(path, buf, size, XATTR_NOFOLLOW)
n, buf = _listxattr_inner(func, path)
return [os.fsdecode(name) for name in split_string0(buf[:n])]
return [os.fsdecode(name) for name in split_string0(buf[:n]) if name]
def getxattr(path, name, *, follow_symlinks=True):
def func(path, name, buf, size):
@ -317,7 +317,7 @@ elif sys.platform.startswith('freebsd'): # pragma: freebsd only
return libc.extattr_list_link(path, ns, buf, size)
n, buf = _listxattr_inner(func, path)
return [os.fsdecode(name) for name in split_lstring(buf[:n])]
return [os.fsdecode(name) for name in split_lstring(buf[:n]) if name]
def getxattr(path, name, *, follow_symlinks=True):
def func(path, name, buf, size):

2
tox.ini
View File

@ -9,7 +9,7 @@ deps =
-rrequirements.d/development.txt
-rrequirements.d/attic.txt
-rrequirements.d/fuse.txt
commands = py.test -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
commands = py.test -n 8 -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
# fakeroot -u needs some env vars:
passenv = *