From 5cc25d986a8561ebabbc9be641897771d3047e58 Mon Sep 17 00:00:00 2001 From: Radek Podgorny Date: Thu, 29 Oct 2015 02:37:43 +0100 Subject: [PATCH 001/321] move away from RawConfigParser to ConfigParser this is a recommended thing since direct use of RawConfigParser is not deprecated according to python docs. --- borg/cache.py | 4 ++-- borg/repository.py | 10 +++++----- borg/testsuite/archiver.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/borg/cache.py b/borg/cache.py index 729149c2..ac53761f 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -112,7 +112,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" os.makedirs(self.path) with open(os.path.join(self.path, 'README'), 'w') as fd: fd.write('This is a Borg cache') - config = configparser.RawConfigParser() + config = configparser.ConfigParser(interpolation=None) config.add_section('cache') config.set('cache', 'version', '1') config.set('cache', 'repository', hexlify(self.repository.id).decode('ascii')) @@ -132,7 +132,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" shutil.rmtree(self.path) def _do_open(self): - self.config = configparser.RawConfigParser() + self.config = configparser.ConfigParser(interpolation=None) config_path = os.path.join(self.path, 'config') self.config.read(config_path) try: diff --git a/borg/repository.py b/borg/repository.py index 69ced28d..97151770 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -1,4 +1,4 @@ -from configparser import RawConfigParser +from configparser import ConfigParser from binascii import hexlify from itertools import islice import errno @@ -79,11 +79,11 @@ class Repository: with open(os.path.join(path, 'README'), 'w') as fd: fd.write('This is a Borg repository\n') os.mkdir(os.path.join(path, 'data')) - config = RawConfigParser() + config = ConfigParser(interpolation=None) config.add_section('repository') config.set('repository', 'version', '1') - config.set('repository', 'segments_per_dir', self.DEFAULT_SEGMENTS_PER_DIR) - config.set('repository', 'max_segment_size', self.DEFAULT_MAX_SEGMENT_SIZE) + config.set('repository', 'segments_per_dir', str(self.DEFAULT_SEGMENTS_PER_DIR)) + config.set('repository', 'max_segment_size', str(self.DEFAULT_MAX_SEGMENT_SIZE)) config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii')) self.save_config(path, config) @@ -136,7 +136,7 @@ class Repository: if not os.path.isdir(path): raise self.DoesNotExist(path) self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive).acquire() - self.config = RawConfigParser() + self.config = ConfigParser(interpolation=None) self.config.read(os.path.join(self.path, 'config')) if 'repository' not in self.config.sections() or self.config.getint('repository', 'version') != 1: raise self.InvalidRepository(path) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 09cdc56a..3ed0fe02 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1,5 +1,5 @@ from binascii import hexlify -from configparser import RawConfigParser +from configparser import ConfigParser import errno import os from io import StringIO @@ -318,7 +318,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): return Repository(self.repository_path).id def _set_repository_id(self, path, id): - config = RawConfigParser() + config = ConfigParser(interpolation=None) config.read(os.path.join(path, 'config')) config.set('repository', 'id', hexlify(id).decode('ascii')) with open(os.path.join(path, 'config'), 'w') as fd: From 3c2dee6eb640a94572c14b30277c4fc7185d37ad Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 31 Oct 2015 19:50:53 +0100 Subject: [PATCH 002/321] vagrant: fix msgpack installation on centos, fixes #342 --- Vagrantfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 90864f58..c67394b4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -42,6 +42,8 @@ def packages_redhatted yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig usermod -a -G fuse vagrant yum install -y fakeroot gcc git patch + # needed to compile msgpack-python (otherwise it will use slow fallback code): + yum install -y gcc-c++ # for building python: yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz-devel sqlite-devel #yum install -y python-pip From 3490469734bbff22f6883295cd9094c1bf6c5bfd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 31 Oct 2015 20:37:21 +0100 Subject: [PATCH 003/321] emit a warning if we have a slow msgpack installed the reason for a slow msgpack can be: - pip install: missing compiler (gcc) - pip install: missing compiler parts (e.g. gcc-c++) - pip install: cached wheel package that was built while the compiler wasn't present - distribution package: badly built msgpack package --- borg/archiver.py | 4 +++- borg/helpers.py | 6 ++++++ borg/testsuite/helpers.py | 14 +++++++++++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 3ab15695..66ebccc1 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -19,7 +19,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, \ + is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .logger import create_logger, setup_logging logger = create_logger() @@ -1015,6 +1015,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask update_excludes(args) + if is_slow_msgpack(): + logger.warning("Using a pure-python msgpack! This will result in lower performance.") return args.func(args) diff --git a/borg/helpers.py b/borg/helpers.py index c9ffc00b..63ef4cfa 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -43,6 +43,8 @@ if have_cython(): from . import crypto import msgpack +import msgpack.fallback + # return codes returned by borg command # when borg is killed by signal N, rc = 128 + N @@ -791,3 +793,7 @@ def int_to_bigint(value): if value.bit_length() > 63: return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True) return value + + +def is_slow_msgpack(): + return msgpack.Packer is msgpack.fallback.Packer diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index a27b2271..2faa569d 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -7,9 +7,10 @@ import os import pytest import sys import msgpack +import msgpack.fallback from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, \ + prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams from . import BaseTestCase @@ -480,3 +481,14 @@ def test_file_size_precision(): assert format_file_size(1234, precision=1) == '1.2 kB' # rounded down assert format_file_size(1254, precision=1) == '1.3 kB' # rounded up assert format_file_size(999990000, precision=1) == '1.0 GB' # and not 999.9 MB or 1000.0 MB + + +def test_is_slow_msgpack(): + saved_packer = msgpack.Packer + try: + msgpack.Packer = msgpack.fallback.Packer + assert is_slow_msgpack() + finally: + msgpack.Packer = saved_packer + # this assumes that we have fast msgpack on test platform: + assert not is_slow_msgpack() From 762fdaadd89e165d101822e933b7fba2d88ddcc7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 31 Oct 2015 22:23:32 +0100 Subject: [PATCH 004/321] prettier error messages, fixes #57 subclasses of "Error": do not show traceback (this is used when a failure is expected and has rather trivial reasons and usually does not need debugging) subclasses of "ErrorWithTraceback": show a traceback (this is for severe and rather unexpected stuff, like consistency / corruption issues or stuff that might need debugging) I reviewed all the Error subclasses whether they fit into the one or other class. Also: fixed docstring typo, docstring formatting --- borg/archiver.py | 4 +++- borg/cache.py | 3 +-- borg/helpers.py | 9 ++++++++- borg/key.py | 9 +++------ borg/locking.py | 10 +++++----- borg/remote.py | 2 +- borg/repository.py | 8 ++++---- 7 files changed, 25 insertions(+), 20 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 3ab15695..3fdad7df 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1062,7 +1062,9 @@ def main(): # pragma: no cover msg = None exit_code = archiver.run(sys.argv[1:]) except Error as e: - msg = e.get_message() + "\n%s" % traceback.format_exc() + msg = e.get_message() + if e.traceback: + msg += "\n%s" % traceback.format_exc() exit_code = e.exit_code except RemoteRepository.RPCError as e: msg = 'Remote Exception.\n%s' % str(e) diff --git a/borg/cache.py b/borg/cache.py index 729149c2..436c625a 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -35,8 +35,7 @@ class Cache: """Repository access aborted""" class EncryptionMethodMismatch(Error): - """Repository encryption method changed since last acccess, refusing to continue - """ + """Repository encryption method changed since last access, refusing to continue""" def __init__(self, repository, key, manifest, path=None, sync=True, do_files=False, warn_if_unencrypted=True): self.lock = None diff --git a/borg/helpers.py b/borg/helpers.py index c9ffc00b..90e0ec19 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -58,12 +58,19 @@ class Error(Exception): # exception handler (that exits short after with the given exit_code), # it is always a (fatal and abrupt) EXIT_ERROR, never just a warning. exit_code = EXIT_ERROR + # show a traceback? + traceback = False def get_message(self): return type(self).__doc__.format(*self.args) -class IntegrityError(Error): +class ErrorWithTraceback(Error): + """like Error, but show a traceback also""" + traceback = True + + +class IntegrityError(ErrorWithTraceback): """Data integrity error""" diff --git a/borg/key.py b/borg/key.py index a9ceef41..43f6c5ca 100644 --- a/borg/key.py +++ b/borg/key.py @@ -19,18 +19,15 @@ PREFIX = b'\0' * 8 class UnsupportedPayloadError(Error): - """Unsupported payload type {}. A newer version is required to access this repository. - """ + """Unsupported payload type {}. A newer version is required to access this repository.""" class KeyfileNotFoundError(Error): - """No key file for repository {} found in {}. - """ + """No key file for repository {} found in {}.""" class RepoKeyNotFoundError(Error): - """No key entry found in the config of repository {}. - """ + """No key entry found in the config of repository {}.""" class HMAC(hmac.HMAC): diff --git a/borg/locking.py b/borg/locking.py index aff3c5fc..159e3c57 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -4,7 +4,7 @@ import os import socket import time -from borg.helpers import Error +from borg.helpers import Error, ErrorWithTraceback ADD, REMOVE = 'add', 'remove' SHARED, EXCLUSIVE = 'shared', 'exclusive' @@ -76,7 +76,7 @@ class TimeoutTimer: class ExclusiveLock: """An exclusive Lock based on mkdir fs operation being atomic""" - class LockError(Error): + class LockError(ErrorWithTraceback): """Failed to acquire the lock {}.""" class LockTimeout(LockError): @@ -85,7 +85,7 @@ class ExclusiveLock: class LockFailed(LockError): """Failed to create/acquire the lock {} ({}).""" - class UnlockError(Error): + class UnlockError(ErrorWithTraceback): """Failed to release the lock {}.""" class NotLocked(UnlockError): @@ -215,10 +215,10 @@ class UpgradableLock: noone is allowed reading) and read access to a resource needs a shared lock (multiple readers are allowed). """ - class SharedLockFailed(Error): + class SharedLockFailed(ErrorWithTraceback): """Failed to acquire shared lock [{}]""" - class ExclusiveLockFailed(Error): + class ExclusiveLockFailed(ErrorWithTraceback): """Failed to acquire write lock [{}]""" def __init__(self, path, exclusive=False, sleep=None, id=None): diff --git a/borg/remote.py b/borg/remote.py index 5d8c71a8..cffa62d7 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -28,7 +28,7 @@ class PathNotAllowed(Error): class InvalidRPCMethod(Error): - """RPC method is not valid""" + """RPC method {} is not valid""" class RepositoryServer: # pragma: no cover diff --git a/borg/repository.py b/borg/repository.py index 69ced28d..acb57e0d 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -11,7 +11,7 @@ import struct import sys from zlib import crc32 -from .helpers import Error, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython +from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython if have_cython(): from .hashindex import NSIndex from .locking import UpgradableLock @@ -45,12 +45,12 @@ class Repository: """Repository {} already exists.""" class InvalidRepository(Error): - """{} is not a valid repository.""" + """{} is not a valid repository. Check repo config.""" - class CheckNeeded(Error): + class CheckNeeded(ErrorWithTraceback): """Inconsistency detected. Please run "borg check {}".""" - class ObjectNotFound(Error): + class ObjectNotFound(ErrorWithTraceback): """Object with key {} not found in repository {}.""" def __init__(self, path, create=False, exclusive=False): From 17403847633eab53e49ba99d0a45808a4621c3be Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 31 Oct 2015 22:41:08 +0100 Subject: [PATCH 005/321] prettier connection closed message, fixes #307 --- borg/remote.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/borg/remote.py b/borg/remote.py index 5d8c71a8..ff814482 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -23,6 +23,10 @@ class ConnectionClosed(Error): """Connection closed by remote host""" +class ConnectionClosedWithHint(ConnectionClosed): + """Connection closed by remote host. {}""" + + class PathNotAllowed(Error): """Repository path not allowed""" @@ -148,7 +152,7 @@ class RemoteRepository: try: version = self.call('negotiate', 1) except ConnectionClosed: - raise Exception('Server immediately closed connection - is Borg installed and working on the server?') + raise ConnectionClosedWithHint('Is borg working on the server?') if version != 1: raise Exception('Server insisted on using unsupported protocol version %d' % version) self.id = self.call('open', location.path, create) From 7ea701f6f755097d0ff941a83030b915760a6bc8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 00:01:23 +0100 Subject: [PATCH 006/321] fix test failure for borg.exe fatal error is rc 2 now (EXIT_ERROR) --- borg/testsuite/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 3ed0fe02..78497e42 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -384,7 +384,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self._set_repository_id(self.repository_path, repository_id) self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) if self.FORK_DEFAULT: - self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=1) # fails + self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=EXIT_ERROR) else: self.assert_raises(Cache.EncryptionMethodMismatch, lambda: self.cmd('create', self.repository_location + '::test.2', 'input')) @@ -397,7 +397,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): shutil.rmtree(self.repository_path + '_encrypted') os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') if self.FORK_DEFAULT: - self.cmd('create', self.repository_location + '_encrypted::test.2', 'input', exit_code=1) # fails + self.cmd('create', self.repository_location + '_encrypted::test.2', 'input', exit_code=EXIT_ERROR) else: self.assert_raises(Cache.RepositoryAccessAborted, lambda: self.cmd('create', self.repository_location + '_encrypted::test.2', 'input')) From 4bf8c8a6f8c0841e241c3d11e8c895fc668cbfe3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 00:40:32 +0100 Subject: [PATCH 007/321] separate parse_args() from run() parse_args concentrates on only processing arguments, including pre and post processing. this needs to be called before run(), which is now receiving the return value of parse_args. this was done so we can have the parsed args outside of the run function, e.g. in main(). --- borg/archiver.py | 32 +++++++++++++++++--------------- borg/testsuite/archiver.py | 3 ++- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index ff475f37..c5268cb5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -985,7 +985,21 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='additional help on TOPIC') return parser - def run(self, args=None): + def parse_args(self, args=None): + # We can't use argparse for "serve" since we don't want it to show up in "Available commands" + if args: + args = self.preprocess_args(args) + parser = self.build_parser(args) + args = parser.parse_args(args or ['-h']) + update_excludes(args) + return args + + def run(self, args): + os.umask(args.umask) # early, before opening files + self.verbose = args.verbose + RemoteRepository.remote_path = args.remote_path + RemoteRepository.umask = args.umask + setup_logging() check_extension_modules() keys_dir = get_keys_dir() if not os.path.exists(keys_dir): @@ -1002,19 +1016,6 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") # For information about cache directory tags, see: # http://www.brynosaurus.com/cachedir/ """).lstrip()) - - # We can't use argparse for "serve" since we don't want it to show up in "Available commands" - if args: - args = self.preprocess_args(args) - parser = self.build_parser(args) - - args = parser.parse_args(args or ['-h']) - self.verbose = args.verbose - setup_logging() - os.umask(args.umask) - RemoteRepository.remote_path = args.remote_path - RemoteRepository.umask = args.umask - update_excludes(args) if is_slow_msgpack(): logger.warning("Using a pure-python msgpack! This will result in lower performance.") return args.func(args) @@ -1062,7 +1063,8 @@ def main(): # pragma: no cover archiver = Archiver() try: msg = None - exit_code = archiver.run(sys.argv[1:]) + args = archiver.parse_args(sys.argv[1:]) + exit_code = archiver.run(args) except Error as e: msg = e.get_message() if e.traceback: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 78497e42..8b30e4fe 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -93,7 +93,8 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): sys.stdout = sys.stderr = output = StringIO() if archiver is None: archiver = Archiver() - ret = archiver.run(list(args)) + args = archiver.parse_args(list(args)) + ret = archiver.run(args) return ret, output.getvalue() finally: sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr From 4a1c99524436f870782c67929eedf4723637c90e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 00:53:55 +0100 Subject: [PATCH 008/321] add --show-rc option enable "terminating with X status, rc N" output, fixes #351 this is needed for tools like borgweb (or in general: when the rc value / exit status should be logged for later review or directly seen on screen). this is off by default, so the output is less verbose (and also does not fail tests which counts lines). --- borg/archiver.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index c5268cb5..50a4d701 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -571,6 +571,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='verbose output') + common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, + help='show/log the return code (rc)') common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', help='do not load/update the file metadata cache used to detect unchanged files') common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M', @@ -1061,9 +1063,9 @@ def main(): # pragma: no cover sys.stderr = io.TextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) setup_signal_handlers() archiver = Archiver() + msg = None + args = archiver.parse_args(sys.argv[1:]) try: - msg = None - args = archiver.parse_args(sys.argv[1:]) exit_code = archiver.run(args) except Error as e: msg = e.get_message() @@ -1081,16 +1083,17 @@ def main(): # pragma: no cover exit_code = EXIT_ERROR if msg: logger.error(msg) - exit_msg = 'terminating with %s status, rc %d' - if exit_code == EXIT_SUCCESS: - logger.info(exit_msg % ('success', exit_code)) - elif exit_code == EXIT_WARNING: - logger.warning(exit_msg % ('warning', exit_code)) - elif exit_code == EXIT_ERROR: - logger.error(exit_msg % ('error', exit_code)) - else: - # if you see 666 in output, it usually means exit_code was None - logger.error(exit_msg % ('abnormal', exit_code or 666)) + if args.show_rc: + exit_msg = 'terminating with %s status, rc %d' + if exit_code == EXIT_SUCCESS: + logger.info(exit_msg % ('success', exit_code)) + elif exit_code == EXIT_WARNING: + logger.warning(exit_msg % ('warning', exit_code)) + elif exit_code == EXIT_ERROR: + logger.error(exit_msg % ('error', exit_code)) + else: + # if you see 666 in output, it usually means exit_code was None + logger.error(exit_msg % ('abnormal', exit_code or 666)) sys.exit(exit_code) From 36900051c5214b0a0a2f0b2f7d39589654afbad5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 19:10:50 +0100 Subject: [PATCH 009/321] move test utilities to testsuite package, add FakeInputs utility --- borg/testsuite/__init__.py | 47 ++++++++++++++++++++++++++++++++++++++ borg/testsuite/archiver.py | 31 +------------------------ 2 files changed, 48 insertions(+), 30 deletions(-) diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 2d2ee904..3af1738b 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -103,3 +103,50 @@ class BaseTestCase(unittest.TestCase): return time.sleep(.1) raise Exception('wait_for_mount(%s) timeout' % path) + + +class changedir: + def __init__(self, dir): + self.dir = dir + + def __enter__(self): + self.old = os.getcwd() + os.chdir(self.dir) + + def __exit__(self, *args, **kw): + os.chdir(self.old) + + +class environment_variable: + def __init__(self, **values): + self.values = values + self.old_values = {} + + def __enter__(self): + for k, v in self.values.items(): + self.old_values[k] = os.environ.get(k) + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + def __exit__(self, *args, **kw): + for k, v in self.old_values.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +class FakeInputs: + """Simulate multiple user inputs, can be used as input() replacement""" + def __init__(self, inputs): + self.inputs = inputs + + def __call__(self, prompt=None): + if prompt is not None: + print(prompt, end='') + try: + return self.inputs.pop(0) + except IndexError: + raise EOFError diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8b30e4fe..028f694f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -23,7 +23,7 @@ from ..crypto import bytes_to_long, num_aes_blocks from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, st_atime_ns, st_mtime_ns from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository -from . import BaseTestCase +from . import BaseTestCase, changedir, environment_variable try: import llfuse @@ -42,35 +42,6 @@ except NameError: PermissionError = OSError -class changedir: - def __init__(self, dir): - self.dir = dir - - def __enter__(self): - self.old = os.getcwd() - os.chdir(self.dir) - - def __exit__(self, *args, **kw): - os.chdir(self.old) - - -class environment_variable: - def __init__(self, **values): - self.values = values - self.old_values = {} - - def __enter__(self): - for k, v in self.values.items(): - self.old_values[k] = os.environ.get(k) - os.environ[k] = v - - def __exit__(self, *args, **kw): - for k, v in self.old_values.items(): - if v is None: - del os.environ[k] - else: - os.environ[k] = v - def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): if fork: try: From 0a6e6cfe2e6e28644d139ac6662e122f2f63da40 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 19:18:29 +0100 Subject: [PATCH 010/321] refactor confirmation code, reduce code duplication, add tests --- borg/archiver.py | 32 ++++++++-------- borg/cache.py | 29 ++++++-------- borg/helpers.py | 81 +++++++++++++++++++++++++++++++++++++++ borg/testsuite/helpers.py | 78 ++++++++++++++++++++++++++++++++++++- 4 files changed, 184 insertions(+), 36 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 50a4d701..f948db79 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -19,7 +19,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, \ + is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .logger import create_logger, setup_logging logger = create_logger() @@ -88,13 +88,12 @@ class Archiver: """Check repository consistency""" repository = self.open_repository(args.repository, exclusive=args.repair) if args.repair: - while not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): - self.print_warning("""'check --repair' is an experimental feature that might result -in data loss. - -Type "Yes I am sure" if you understand this and want to continue.\n""") - if input('Do you want to continue? ') == 'Yes I am sure': - break + msg = ("'check --repair' is an experimental feature that might result in data loss." + + "\n" + + "Type 'YES' if you understand this and want to continue: ") + if not yes(msg, false_msg="Aborting.", + env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + return EXIT_ERROR if not args.archives_only: logger.info('Starting repository check...') if repository.check(repair=args.repair): @@ -330,15 +329,16 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") logger.info(str(cache)) else: if not args.cache_only: - print("You requested to completely DELETE the repository *including* all archives it contains:", file=sys.stderr) + msg = [] + msg.append("You requested to completely DELETE the repository *including* all archives it contains:") for archive_info in manifest.list_archive_infos(sort_by='ts'): - print(format_archive(archive_info), file=sys.stderr) - if not os.environ.get('BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): - print("""Type "YES" if you understand this and want to continue.\n""", file=sys.stderr) - # XXX: prompt may end up on stdout, but we'll assume that input() does the right thing - if input('Do you want to continue? ') != 'YES': - self.exit_code = EXIT_ERROR - return self.exit_code + msg.append(format_archive(archive_info)) + msg.append("Type 'YES' if you understand this and want to continue: ") + msg = '\n'.join(msg) + if not yes(msg, false_msg="Aborting.", + env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + self.exit_code = EXIT_ERROR + return self.exit_code repository.destroy() logger.info("Repository deleted.") cache.destroy() diff --git a/borg/cache.py b/borg/cache.py index 12c30227..c3f085cd 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -14,7 +14,7 @@ from .key import PlaintextKey from .logger import create_logger logger = create_logger() from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \ - bigint_to_int, format_file_size, have_cython + bigint_to_int, format_file_size, have_cython, yes from .locking import UpgradableLock from .hashindex import ChunkIndex @@ -51,15 +51,21 @@ class Cache: # Warn user before sending data to a never seen before unencrypted repository if not os.path.exists(self.path): if warn_if_unencrypted and isinstance(key, PlaintextKey): - if not self._confirm('Warning: Attempting to access a previously unknown unencrypted repository', - 'BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): + msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" + + "\n" + + "Do you want to continue? [yN] ") + if not yes(msg, false_msg="Aborting.", + env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): raise self.CacheInitAbortedError() self.create() self.open() # Warn user before sending data to a relocated repository if self.previous_location and self.previous_location != repository._location.canonical_path(): - msg = 'Warning: The repository at location {} was previously located at {}'.format(repository._location.canonical_path(), self.previous_location) - if not self._confirm(msg, 'BORG_RELOCATED_REPO_ACCESS_IS_OK'): + msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + + "\n" + + "Do you want to continue? [yN] ") + if not yes(msg, false_msg="Aborting.", + env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): raise self.RepositoryAccessAborted() if sync and self.manifest.id != self.manifest_id: @@ -92,19 +98,6 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" stats[field] = format_file_size(stats[field]) return Summary(**stats) - def _confirm(self, message, env_var_override=None): - print(message, file=sys.stderr) - if env_var_override and os.environ.get(env_var_override): - print("Yes (From {})".format(env_var_override), file=sys.stderr) - return True - if not sys.stdin.isatty(): - return False - try: - answer = input('Do you want to continue? [yN] ') - except EOFError: - return False - return answer and answer in 'Yy' - def create(self): """Create a new empty cache at `self.path` """ diff --git a/borg/helpers.py b/borg/helpers.py index 6a170596..a2941ddd 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -804,3 +804,84 @@ def int_to_bigint(value): def is_slow_msgpack(): return msgpack.Packer is msgpack.fallback.Packer + + +def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, + default=False, default_notty=None, default_eof=None, + falsish=('No', 'no', 'N', 'n'), truish=('Yes', 'yes', 'Y', 'y'), + env_var_override=None, ifile=None, ofile=None, input=input): + """ + Output (usually a question) and let user input an answer. + Qualifies the answer according to falsish and truish as True or False. + If it didn't qualify and retry_msg is None (no retries wanted), + return the default [which defaults to False]. Otherwise let user retry + answering until answer is qualified. + + If env_var_override is given and it is non-empty, counts as truish answer + and won't ask user for an answer. + If we don't have a tty as input and default_notty is not None, return its value. + Otherwise read input from non-tty and proceed as normal. + If EOF is received instead an input, return default_eof [or default, if not given]. + + :param msg: introducing message to output on ofile, no \n is added [None] + :param retry_msg: retry message to output on ofile, no \n is added [None] + (also enforces retries instead of returning default) + :param false_msg: message to output before returning False [None] + :param true_msg: message to output before returning True [None] + :param default: default return value (empty answer is given) [False] + :param default_notty: if not None, return its value if no tty is connected [None] + :param default_eof: return value if EOF was read as answer [same as default] + :param falsish: sequence of answers qualifying as False + :param truish: sequence of answers qualifying as True + :param env_var_override: environment variable name [None] + :param ifile: input stream [sys.stdin] (only for testing!) + :param ofile: output stream [sys.stderr] + :param input: input function [input from builtins] + :return: boolean answer value, True or False + """ + # note: we do not assign sys.stdin/stderr as defaults above, so they are + # really evaluated NOW, not at function definition time. + if ifile is None: + ifile = sys.stdin + if ofile is None: + ofile = sys.stderr + if default not in (True, False): + raise ValueError("invalid default value, must be True or False") + if default_notty not in (None, True, False): + raise ValueError("invalid default_notty value, must be None, True or False") + if default_eof not in (None, True, False): + raise ValueError("invalid default_eof value, must be None, True or False") + if msg: + print(msg, file=ofile, end='') + ofile.flush() + if env_var_override: + value = os.environ.get(env_var_override) + # currently, any non-empty value counts as truish + # TODO: change this so one can give y/n there? + if value: + value = bool(value) + value_str = truish[0] if value else falsish[0] + print("{} (from {})".format(value_str, env_var_override), file=ofile) + return value + if default_notty is not None and not ifile.isatty(): + # looks like ifile is not a terminal (but e.g. a pipe) + return default_notty + while True: + try: + answer = input() # XXX how can we use ifile? + except EOFError: + return default_eof if default_eof is not None else default + if answer in truish: + if true_msg: + print(true_msg, file=ofile) + return True + if answer in falsish: + if false_msg: + print(false_msg, file=ofile) + return False + if retry_msg is None: + # no retries wanted, we just return the default + return default + if retry_msg: + print(retry_msg, file=ofile, end='') + ofile.flush() diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 2faa569d..58861aaa 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -10,9 +10,9 @@ import msgpack import msgpack.fallback from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, \ + prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams -from . import BaseTestCase +from . import BaseTestCase, environment_variable, FakeInputs class BigIntTestCase(BaseTestCase): @@ -492,3 +492,77 @@ def test_is_slow_msgpack(): msgpack.Packer = saved_packer # this assumes that we have fast msgpack on test platform: assert not is_slow_msgpack() + + +def test_yes_simple(): + input = FakeInputs(['y', 'Y', 'yes', 'Yes', ]) + assert yes(input=input) + assert yes(input=input) + assert yes(input=input) + assert yes(input=input) + input = FakeInputs(['n', 'N', 'no', 'No', ]) + assert not yes(input=input) + assert not yes(input=input) + assert not yes(input=input) + assert not yes(input=input) + + +def test_yes_custom(): + input = FakeInputs(['YES', 'SURE', 'NOPE', ]) + assert yes(truish=('YES', ), input=input) + assert yes(truish=('SURE', ), input=input) + assert not yes(falsish=('NOPE', ), input=input) + + +def test_yes_env(): + input = FakeInputs(['n', 'n']) + with environment_variable(OVERRIDE_THIS='nonempty'): + assert yes(env_var_override='OVERRIDE_THIS', input=input) + with environment_variable(OVERRIDE_THIS=None): # env not set + assert not yes(env_var_override='OVERRIDE_THIS', input=input) + + +def test_yes_defaults(): + input = FakeInputs(['invalid', '', ' ']) + assert not yes(input=input) # default=False + assert not yes(input=input) + assert not yes(input=input) + input = FakeInputs(['invalid', '', ' ']) + assert yes(default=True, input=input) + assert yes(default=True, input=input) + assert yes(default=True, input=input) + ifile = StringIO() + assert yes(default_notty=True, ifile=ifile) + assert not yes(default_notty=False, ifile=ifile) + input = FakeInputs([]) + assert yes(default_eof=True, input=input) + assert not yes(default_eof=False, input=input) + with pytest.raises(ValueError): + yes(default=None) + with pytest.raises(ValueError): + yes(default_notty='invalid') + with pytest.raises(ValueError): + yes(default_eof='invalid') + + +def test_yes_retry(): + input = FakeInputs(['foo', 'bar', 'y', ]) + assert yes(retry_msg='Retry: ', input=input) + input = FakeInputs(['foo', 'bar', 'N', ]) + assert not yes(retry_msg='Retry: ', input=input) + + +def test_yes_output(capfd): + input = FakeInputs(['invalid', 'y', 'n']) + assert yes(msg='intro-msg', false_msg='false-msg', true_msg='true-msg', retry_msg='retry-msg', input=input) + out, err = capfd.readouterr() + assert out == '' + assert 'intro-msg' in err + assert 'retry-msg' in err + assert 'true-msg' in err + assert not yes(msg='intro-msg', false_msg='false-msg', true_msg='true-msg', retry_msg='retry-msg', input=input) + out, err = capfd.readouterr() + assert out == '' + assert 'intro-msg' in err + assert 'retry-msg' not in err + assert 'false-msg' in err From 8493cfb0f1f758be0595dea8063e4bd1ec786cff Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 22:18:47 +0100 Subject: [PATCH 011/321] fix .coverragerc omits seems like they need a */ prefix also: exclude everything below support/ - this is 3rd party stuff. --- .coveragerc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 7c4ccf9e..0ae90754 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,10 +2,10 @@ branch = True source = borg omit = - borg/__init__.py - borg/__main__.py - borg/_version.py - borg/support/*.py + */borg/__init__.py + */borg/__main__.py + */borg/_version.py + */borg/support/* [report] exclude_lines = From d7b6cc352726405fd81bf6ff6cef969bde16f8f0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 22:36:18 +0100 Subject: [PATCH 012/321] fix .coverragerc path as the cwd is in toxworkdir, .coveragerc is in ../ --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c260b506..a000e1af 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,6 @@ changedir = {toxworkdir} deps = -rrequirements.d/development.txt attic -commands = py.test --cov=borg --benchmark-skip --pyargs {posargs:borg.testsuite} +commands = py.test --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = * From 35280e9a6507c90c224c7e67e10919fad396a7ca Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 23:06:52 +0100 Subject: [PATCH 013/321] label platform code, exclude freebsd and unknown platform from coverage measurement this coverage configuration is mostly for travis and there we only can test linux and darwin. --- .coveragerc | 2 ++ borg/platform.py | 8 ++++---- borg/xattr.py | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.coveragerc b/.coveragerc index 0ae90754..065ae433 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,8 @@ omit = [report] exclude_lines = pragma: no cover + pragma: freebsd only + pragma: unknown platform only def __repr__ raise AssertionError raise NotImplementedError diff --git a/borg/platform.py b/borg/platform.py index caa3b4ed..1bc8ee5e 100644 --- a/borg/platform.py +++ b/borg/platform.py @@ -1,12 +1,12 @@ import sys -if sys.platform.startswith('linux'): +if sys.platform.startswith('linux'): # pragma: linux only from .platform_linux import acl_get, acl_set, API_VERSION -elif sys.platform.startswith('freebsd'): +elif sys.platform.startswith('freebsd'): # pragma: freebsd only from .platform_freebsd import acl_get, acl_set, API_VERSION -elif sys.platform == 'darwin': +elif sys.platform == 'darwin': # pragma: darwin only from .platform_darwin import acl_get, acl_set, API_VERSION -else: +else: # pragma: unknown platform only API_VERSION = 2 def acl_get(path, item, st, numeric_owner=False): diff --git a/borg/xattr.py b/borg/xattr.py index ded6d752..9c80c326 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -36,7 +36,7 @@ def _check(rv, path=None): raise OSError(get_errno(), path) return rv -if sys.platform.startswith('linux'): +if sys.platform.startswith('linux'): # pragma: linux only libc.llistxattr.argtypes = (c_char_p, c_char_p, c_size_t) libc.llistxattr.restype = c_ssize_t libc.flistxattr.argtypes = (c_int, c_char_p, c_size_t) @@ -100,7 +100,7 @@ if sys.platform.startswith('linux'): func = libc.lsetxattr _check(func(path, name, value, len(value) if value else 0, 0), path) -elif sys.platform == 'darwin': +elif sys.platform == 'darwin': # pragma: darwin only libc.listxattr.argtypes = (c_char_p, c_char_p, c_size_t, c_int) libc.listxattr.restype = c_ssize_t libc.flistxattr.argtypes = (c_int, c_char_p, c_size_t) @@ -166,7 +166,7 @@ elif sys.platform == 'darwin': flags = XATTR_NOFOLLOW _check(func(path, name, value, len(value) if value else 0, 0, flags), path) -elif sys.platform.startswith('freebsd'): +elif sys.platform.startswith('freebsd'): # pragma: freebsd only EXTATTR_NAMESPACE_USER = 0x0001 libc.extattr_list_fd.argtypes = (c_int, c_int, c_char_p, c_size_t) libc.extattr_list_fd.restype = c_ssize_t @@ -247,7 +247,7 @@ elif sys.platform.startswith('freebsd'): func = libc.extattr_set_link _check(func(path, EXTATTR_NAMESPACE_USER, name, value, len(value) if value else 0), path) -else: +else: # pragma: unknown platform only def listxattr(path, *, follow_symlinks=True): return [] From ff6627105721bb5946324bc94f1b5aa261012e9d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 23:20:32 +0100 Subject: [PATCH 014/321] only measure coverage for borgbackup code, not for tests --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 065ae433..81595968 100644 --- a/.coveragerc +++ b/.coveragerc @@ -6,6 +6,7 @@ omit = */borg/__main__.py */borg/_version.py */borg/support/* + */borg/testsuite/* [report] exclude_lines = From 23a3e3b0680e99bb690af07dc38c3800390f59da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 1 Nov 2015 23:22:04 +0100 Subject: [PATCH 015/321] do not measure coverage for fuse.py it's always at 0% although we do have fuse tests. guess one can't measure such code with coverage.py. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 81595968..077e3e6f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,6 +5,7 @@ omit = */borg/__init__.py */borg/__main__.py */borg/_version.py + */borg/fuse.py */borg/support/* */borg/testsuite/* From e6231896cded2b293e0b37034ea52d971e6e4d8b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 00:14:01 +0100 Subject: [PATCH 016/321] emit a deprecation warning for --compression N deprecating it in the source is not enough, we also need to tell the users to fix their scripts. --- borg/helpers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/borg/helpers.py b/borg/helpers.py index a2941ddd..c371c7b3 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -405,6 +405,10 @@ def CompressionSpec(s): raise ValueError # DEPRECATED: it is just --compression N if 0 <= compression <= 9: + print('Warning: --compression %d is deprecated, please use --compression zlib,%d.' % (compression, compression)) + if compression == 0: + print('Hint: instead of --compression zlib,0 you could also use --compression none for better performance.') + print('Hint: archives generated using --compression none are not compatible with borg < 0.25.0.') return dict(name='zlib', level=compression) raise ValueError except ValueError: From 36cc37732914f8c7551dab7348401a5608401050 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 01:59:22 +0100 Subject: [PATCH 017/321] use default_notty=False for confirmations, fixes #345 this is so that e.g. cron jobs do not hang indefinitely if yes() is called, but it will just default to "no" if not tty is connected. if you need to enforce a "yes" answer (which is not recommended for the security critical questions), you can use the environment: BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=Y --- borg/archiver.py | 4 ++-- borg/cache.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index f948db79..c2aaecb8 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -91,7 +91,7 @@ class Archiver: msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + "Type 'YES' if you understand this and want to continue: ") - if not yes(msg, false_msg="Aborting.", + if not yes(msg, false_msg="Aborting.", default_notty=False, env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): return EXIT_ERROR if not args.archives_only: @@ -335,7 +335,7 @@ class Archiver: msg.append(format_archive(archive_info)) msg.append("Type 'YES' if you understand this and want to continue: ") msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", + if not yes(msg, false_msg="Aborting.", default_notty=False, env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): self.exit_code = EXIT_ERROR return self.exit_code diff --git a/borg/cache.py b/borg/cache.py index c3f085cd..7eb85405 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -54,7 +54,7 @@ class Cache: msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", + if not yes(msg, false_msg="Aborting.", default_notty=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): raise self.CacheInitAbortedError() self.create() @@ -64,7 +64,7 @@ class Cache: msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", + if not yes(msg, false_msg="Aborting.", default_notty=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): raise self.RepositoryAccessAborted() From 734dae80efbcd929249b41fa76b456b9028fc2e2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 19:47:09 +0100 Subject: [PATCH 018/321] improve chunker params docs, fixes #362 --- docs/internals.rst | 8 +++----- docs/usage.rst | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index d989fd9c..2ebed0c5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -196,6 +196,7 @@ to the archive metadata. A chunk is stored as an object as well, of course. +.. _chunker_details: Chunks ------ @@ -212,16 +213,13 @@ can be used to tune the chunker parameters, the default is: - HASH_MASK_BITS = 16 (statistical medium chunk size ~= 2^16 B = 64 kiB) - HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`) -The default parameters are OK for relatively small backup data volumes and -repository sizes and a lot of available memory (RAM) and disk space for the -chunk index. If that does not apply, you are advised to tune these parameters -to keep the chunk count lower than with the defaults. - The buzhash table is altered by XORing it with a seed randomly generated once for the archive, and stored encrypted in the keyfile. This is to prevent chunk size based fingerprinting attacks on your encrypted repo contents (to guess what files you have based on a specific set of chunk sizes). +For some more general usage hints see also `--chunker-params`. + Indexes / Caches ---------------- diff --git a/docs/usage.rst b/docs/usage.rst index 6b88d5c6..5a7ce0ed 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -391,6 +391,48 @@ Additional Notes Here are misc. notes about topics that are maybe not covered in enough detail in the usage section. +--chunker-params +~~~~~~~~~~~~~~~~ +The chunker params influence how input files are cut into pieces (chunks) +which are then considered for deduplication. They also have a big impact on +resource usage (RAM and disk space) as the amount of resources needed is +(also) determined by the total amount of chunks in the repository (see +`Indexes / Caches memory usage` for details). + +`--chunker-params=10,23,16,4095 (default)` results in a fine-grained deduplication +and creates a big amount of chunks and thus uses a lot of resources to manage them. +This is good for relatively small data volumes and if the machine has a good +amount of free RAM and disk space. + +`--chunker-params=19,23,21,4095` results in a coarse-grained deduplication and +creates a much smaller amount of chunks and thus uses less resources. +This is good for relatively big data volumes and if the machine has a relatively +low amount of free RAM and disk space. + +If you already have made some archives in a repository and you then change +chunker params, this of course impacts deduplication as the chunks will be +cut differently. + +In the worst case (all files are big and were touched in between backups), this +will store all content into the repository again. + +Usually, it is not that bad though: +- usually most files are not touched, so it will just re-use the old chunks +it already has in the repo +- files smaller than the (both old and new) minimum chunksize result in only +one chunk anyway, so the resulting chunks are same and deduplication will apply + +If you switch chunker params to save resources for an existing repo that +already has some backup archives, you will see an increasing effect over time, +when more and more files have been touched and stored again using the bigger +chunksize **and** all references to the smaller older chunks have been removed +(by deleting / pruning archives). + +If you want to see an immediate big effect on resource usage, you better start +a new repository when changing chunker params. + +For more details, see :ref:`chunker_details`. + --read-special ~~~~~~~~~~~~~~ From 6d5cc06cf6ed0d55e5a9eb19c8f999d8b23fc21d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 20:36:13 +0100 Subject: [PATCH 019/321] remove unused imports, add missing imports --- borg/archiver.py | 2 +- borg/cache.py | 4 ---- borg/key.py | 1 + borg/logger.py | 1 - borg/repository.py | 1 - borg/testsuite/logger.py | 1 - setup.py | 1 - 7 files changed, 2 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index c2aaecb8..e18ded6c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,7 +17,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ - get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ + get_cache_dir, get_keys_dir, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR diff --git a/borg/cache.py b/borg/cache.py index 7eb85405..b31f68a4 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -1,14 +1,10 @@ import configparser from .remote import cache_if_remote from collections import namedtuple -import errno import os import stat -import sys from binascii import hexlify import shutil -import tarfile -import tempfile from .key import PlaintextKey from .logger import create_logger diff --git a/borg/key.py b/borg/key.py index 43f6c5ca..bbe8bb58 100644 --- a/borg/key.py +++ b/borg/key.py @@ -2,6 +2,7 @@ from binascii import hexlify, a2b_base64, b2a_base64 import configparser import getpass import os +import sys import textwrap import hmac from hashlib import sha256 diff --git a/borg/logger.py b/borg/logger.py index 69f2a3c2..6c5d2ff8 100644 --- a/borg/logger.py +++ b/borg/logger.py @@ -32,7 +32,6 @@ The way to use this is as follows: import inspect import logging -import sys def setup_logging(stream=None): diff --git a/borg/repository.py b/borg/repository.py index 63dc2793..230c7d86 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -8,7 +8,6 @@ logger = logging.getLogger(__name__) import os import shutil import struct -import sys from zlib import crc32 from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython diff --git a/borg/testsuite/logger.py b/borg/testsuite/logger.py index 1db72bf2..ff8b0a03 100644 --- a/borg/testsuite/logger.py +++ b/borg/testsuite/logger.py @@ -1,7 +1,6 @@ import logging from io import StringIO -from mock import Mock import pytest from ..logger import find_parent_module, create_logger, setup_logging diff --git a/setup.py b/setup.py index 6a5ed586..284fddd0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ from glob import glob from distutils.command.build import build from distutils.core import Command -from distutils.errors import DistutilsOptionError from distutils import log from setuptools.command.build_py import build_py From bdf7dc65bddffb67fcfe70155433a8bf8d5ac4da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 20:53:04 +0100 Subject: [PATCH 020/321] docs: add example for rename --- docs/usage.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index 5a7ce0ed..32345c3b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -257,6 +257,19 @@ Note: currently, extract always writes into the current working directory ("."), .. include:: usage/rename.rst.inc +Examples +~~~~~~~~ +:: + + $ borg create /mnt/backup::archivename ~ + $ borg list /mnt/backup + archivename Mon Nov 2 20:40:06 2015 + + $ borg rename /mnt/backup::archivename newname + $ borg list /mnt/backup + newname Mon Nov 2 20:40:06 2015 + + .. include:: usage/delete.rst.inc .. include:: usage/list.rst.inc From a69f7b0f595e8ab6789f767967f600a12d68704b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 2 Nov 2015 21:06:04 +0100 Subject: [PATCH 021/321] vagrant: use pyinstaller from develop branch, fixes #336 it has some fixes that are not in pyinstaller 3.0 release (and not in master branch). --- Vagrantfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index c67394b4..02b43c91 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -208,7 +208,8 @@ def install_pyinstaller(boxname) . borg-env/bin/activate git clone https://github.com/pyinstaller/pyinstaller.git cd pyinstaller - git checkout master + # use develop branch for now, see borgbackup issue #336 + git checkout develop pip install -e . EOF end @@ -220,7 +221,8 @@ def install_pyinstaller_bootloader(boxname) . borg-env/bin/activate git clone https://github.com/pyinstaller/pyinstaller.git cd pyinstaller - git checkout master + # use develop branch for now, see borgbackup issue #336 + git checkout develop # build bootloader, if it is not included cd bootloader python ./waf all From 22262e3fb712193e830632eadaf85673b9be259b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 00:41:20 +0100 Subject: [PATCH 022/321] add a test to find disk-full issues, #327 --- borg/testsuite/archiver.py | 70 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 028f694f..fc6649d3 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -3,6 +3,7 @@ from configparser import ConfigParser import errno import os from io import StringIO +import random import stat import subprocess import sys @@ -112,6 +113,75 @@ def test_return_codes(cmd, tmpdir): assert rc == EXIT_ERROR # duplicate archive name +""" +test_disk_full is very slow and not recommended to be included in daily testing. +for this test, an empty, writable 16MB filesystem mounted on DF_MOUNT is required. +for speed and other reasons, it is recommended that the underlying block device is +in RAM, not a magnetic or flash disk. + +assuming /tmp is a tmpfs (in memory filesystem), one can use this: +dd if=/dev/zero of=/tmp/borg-disk bs=16M count=1 +mkfs.ext4 /tmp/borg-disk +mkdir /tmp/borg-mount +sudo mount /tmp/borg-disk /tmp/borg-mount + +if the directory does not exist, the test will be skipped. +""" +DF_MOUNT = '/tmp/borg-mount' + +@pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT) +def test_disk_full(cmd): + def make_files(dir, count, size, rnd=True): + shutil.rmtree(dir, ignore_errors=True) + os.mkdir(dir) + if rnd: + count = random.randint(1, count) + size = random.randint(1, size) + for i in range(count): + fn = os.path.join(dir, "file%03d" % i) + with open(fn, 'wb') as f: + data = os.urandom(size) + f.write(data) + + with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='1'): + mount = DF_MOUNT + assert os.path.exists(mount) + repo = os.path.join(mount, 'repo') + input = os.path.join(mount, 'input') + reserve = os.path.join(mount, 'reserve') + for j in range(100): + shutil.rmtree(repo, ignore_errors=True) + rc, out = cmd('init', repo) + print('init', rc, out) + assert rc == EXIT_SUCCESS + # keep some space in reserve that we can free up later: + make_files(reserve, 1, 8000000, rnd=False) + try: + success, i = True, 0 + while success: + i += 1 + make_files(input, 20, 200000) # random, ~1MB + try: + rc, out = cmd('create', '%s::test%03d' % (repo, i), input) + success = rc == EXIT_SUCCESS + if not success: + print('create', rc, out) + finally: + # make sure repo is not locked + shutil.rmtree(os.path.join(repo, 'lock.exclusive'), ignore_errors=True) + os.remove(os.path.join(repo, 'lock.roster')) + finally: + # now some error happened, likely we are out of disk space. + # free some space so we can expect borg to be able to work normally: + shutil.rmtree(reserve, ignore_errors=True) + rc, out = cmd('list', repo) + print('list', rc, out) + assert rc == EXIT_SUCCESS + rc, out = cmd('check', '--repair', repo) + print('check', rc, out) + assert rc == EXIT_SUCCESS + + class ArchiverTestCaseBase(BaseTestCase): EXE = None # python source based FORK_DEFAULT = False From 98f3b8d937dea79e86b0c2f9c608a094d01325eb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 13:46:00 +0100 Subject: [PATCH 023/321] fix "check" for repos that have incomplete chunks, fixes #364 added try/finally (the code in between was just indented, no other code changes) to make sure it sets self.index back to None, even if the code crashes e.g. due to an IntegrityError caused by an incomplete segment caused by a disk full condition. also, in prepare_txn, create an empty in-memory index if transaction_id is None, which is required by the Repository.check code to work correctly. If the index is not empty there, it will miscalculate segment usage (self.segments). --- borg/repository.py | 69 ++++++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 33 deletions(-) diff --git a/borg/repository.py b/borg/repository.py index 230c7d86..bf7a27a1 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -175,7 +175,7 @@ class Repository: # the repository instance lives on - even if exceptions happened. self._active_txn = False raise - if not self.index: + if not self.index or transaction_id is None: self.index = self.open_index(transaction_id) if transaction_id is None: self.segments = {} @@ -237,38 +237,41 @@ class Repository: def replay_segments(self, index_transaction_id, segments_transaction_id): self.prepare_txn(index_transaction_id, do_cleanup=False) - for segment, filename in self.io.segment_iterator(): - if index_transaction_id is not None and segment <= index_transaction_id: - continue - if segment > segments_transaction_id: - break - self.segments[segment] = 0 - for tag, key, offset in self.io.iter_objects(segment): - if tag == TAG_PUT: - try: - s, _ = self.index[key] - self.compact.add(s) - self.segments[s] -= 1 - except KeyError: - pass - self.index[key] = segment, offset - self.segments[segment] += 1 - elif tag == TAG_DELETE: - try: - s, _ = self.index.pop(key) - self.segments[s] -= 1 - self.compact.add(s) - except KeyError: - pass - self.compact.add(segment) - elif tag == TAG_COMMIT: + try: + for segment, filename in self.io.segment_iterator(): + if index_transaction_id is not None and segment <= index_transaction_id: continue - else: - raise self.CheckNeeded(self.path) - if self.segments[segment] == 0: - self.compact.add(segment) - self.write_index() - self.rollback() + if segment > segments_transaction_id: + break + # code duplication below?? vvv (see similar code in check()) + self.segments[segment] = 0 + for tag, key, offset in self.io.iter_objects(segment): + if tag == TAG_PUT: + try: + s, _ = self.index[key] + self.compact.add(s) + self.segments[s] -= 1 + except KeyError: + pass + self.index[key] = segment, offset + self.segments[segment] += 1 + elif tag == TAG_DELETE: + try: + s, _ = self.index.pop(key) + self.segments[s] -= 1 + self.compact.add(s) + except KeyError: + pass + self.compact.add(segment) + elif tag == TAG_COMMIT: + continue + else: + raise self.CheckNeeded(self.path) + if self.segments[segment] == 0: + self.compact.add(segment) + self.write_index() + finally: + self.rollback() def check(self, repair=False): """Check repository consistency @@ -297,7 +300,7 @@ class Repository: if repair: self.io.cleanup(transaction_id) segments_transaction_id = self.io.get_segments_transaction_id() - self.prepare_txn(None) + self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! for segment, filename in self.io.segment_iterator(): if segment > transaction_id: continue From f804c6fb1ca3ece53fbf62f3c80161808fc54811 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 13:52:26 +0100 Subject: [PATCH 024/321] repository check code: added some comments --- borg/repository.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/borg/repository.py b/borg/repository.py index bf7a27a1..e1651be2 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -178,8 +178,8 @@ class Repository: if not self.index or transaction_id is None: self.index = self.open_index(transaction_id) if transaction_id is None: - self.segments = {} - self.compact = set() + self.segments = {} # XXX bad name: usage_count_of_segment_x = self.segments[x] + self.compact = set() # XXX bad name: segments_needing_compaction = self.compact else: if do_cleanup: self.io.cleanup(transaction_id) @@ -335,12 +335,15 @@ class Repository: continue else: report_error('Unexpected tag {} in segment {}'.format(tag, segment)) + # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found if repair and segments_transaction_id is None: report_error('Adding commit tag to segment {}'.format(transaction_id)) self.io.segment = transaction_id + 1 self.io.write_commit() if current_index and not repair: + # current_index = "as found on disk" + # self.index = "as rebuilt in-memory from segments" if len(current_index) != len(self.index): report_error('Index object count mismatch. {} != {}'.format(len(current_index), len(self.index))) elif current_index: From f28d5d1f96ea0194f39ce9d380f1342fa9c456af Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 19:39:05 +0100 Subject: [PATCH 025/321] disk full test: some improvements - can create 0-byte files now - frees space early (avoids running out of disk space at repo init time) - creates multiple reserve files, so we do not only reserve some space, but also some inodes - only print output if there is an error RC - if make_files makes us run out of space, that is not interesting, just start a new iteration from scratch --- borg/testsuite/archiver.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index fc6649d3..61d954e6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -136,7 +136,8 @@ def test_disk_full(cmd): os.mkdir(dir) if rnd: count = random.randint(1, count) - size = random.randint(1, size) + if size > 1: + size = random.randint(1, size) for i in range(count): fn = os.path.join(dir, "file%03d" % i) with open(fn, 'wb') as f: @@ -151,16 +152,24 @@ def test_disk_full(cmd): reserve = os.path.join(mount, 'reserve') for j in range(100): shutil.rmtree(repo, ignore_errors=True) + shutil.rmtree(input, ignore_errors=True) + # keep some space and some inodes in reserve that we can free up later: + make_files(reserve, 80, 100000, rnd=False) rc, out = cmd('init', repo) - print('init', rc, out) + if rc != EXIT_SUCCESS: + print('init', rc, out) assert rc == EXIT_SUCCESS - # keep some space in reserve that we can free up later: - make_files(reserve, 1, 8000000, rnd=False) try: success, i = True, 0 while success: i += 1 - make_files(input, 20, 200000) # random, ~1MB + try: + make_files(input, 20, 200000) + except OSError as err: + if err.errno == errno.ENOSPC: + # already out of space + break + raise try: rc, out = cmd('create', '%s::test%03d' % (repo, i), input) success = rc == EXIT_SUCCESS @@ -175,10 +184,11 @@ def test_disk_full(cmd): # free some space so we can expect borg to be able to work normally: shutil.rmtree(reserve, ignore_errors=True) rc, out = cmd('list', repo) - print('list', rc, out) - assert rc == EXIT_SUCCESS + if rc != EXIT_SUCCESS: + print('list', rc, out) rc, out = cmd('check', '--repair', repo) - print('check', rc, out) + if rc != EXIT_SUCCESS: + print('check', rc, out) assert rc == EXIT_SUCCESS From fa35525b5861697a3fce75000d69165b2b39d999 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 19:52:49 +0100 Subject: [PATCH 026/321] create from stdin: also save atime, ctime cosmetic, just for completeness. --- borg/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index 2333a102..86f67136 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -483,13 +483,14 @@ Number of files: {0.stats.nfiles}'''.format(self) for chunk in self.chunker.chunkify(fd): chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)) self.stats.nfiles += 1 + t = int_to_bigint(int(time.time()) * 1000000000) item = { b'path': path, b'chunks': chunks, b'mode': 0o100660, # regular file, ug=rw b'uid': uid, b'user': uid2user(uid), b'gid': gid, b'group': gid2group(gid), - b'mtime': int_to_bigint(int(time.time()) * 1000000000) + b'mtime': t, b'atime': t, b'ctime': t, } self.add_item(item) return 'i' # stdin From 12b5d07e559f1487d4e8de834b16f5210a4e7c5c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 20:21:52 +0100 Subject: [PATCH 027/321] fix RobustUnpacker, it missed some metadata keys. add check for unknown metadata keys. not just the new atime and ctime keys were missing, but also bsdflags. --- borg/archive.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index 86f67136..a5db1d69 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -217,6 +217,9 @@ Number of files: {0.stats.nfiles}'''.format(self) yield item def add_item(self, item): + unknown_keys = set(item) - ITEM_KEYS + assert not unknown_keys, ('unknown item metadata keys detected, please update ITEM_KEYS: %s', + ','.join(k.decode('ascii') for k in unknown_keys)) if self.show_progress and time.time() - self.last_progress > 0.2: self.stats.show_progress(item=item) self.last_progress = time.time() @@ -589,10 +592,17 @@ Number of files: {0.stats.nfiles}'''.format(self) return Archive._open_rb(path, st) +# this set must be kept complete, otherwise the RobustUnpacker might malfunction: +ITEM_KEYS = set([b'path', b'source', b'rdev', b'chunks', + b'mode', b'user', b'group', b'uid', b'gid', b'mtime', b'atime', b'ctime', + b'xattrs', b'bsdflags', + ]) + + class RobustUnpacker: """A restartable/robust version of the streaming msgpack unpacker """ - item_keys = [msgpack.packb(name) for name in ('path', 'mode', 'source', 'chunks', 'rdev', 'xattrs', 'user', 'group', 'uid', 'gid', 'mtime')] + item_keys = [msgpack.packb(name) for name in ITEM_KEYS] def __init__(self, validator): super().__init__() From d2b7dbc0a8c3a95baac2734df375e156caedadd3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 22:51:59 +0100 Subject: [PATCH 028/321] debugging helper: borg debug-dump-archive-items --- borg/archiver.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index e18ded6c..145ca1e6 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -500,6 +500,20 @@ class Archiver: print("warning: %s" % e) return self.exit_code + def do_debug_dump_archive_items(self, args): + """dump (decrypted, decompressed) archive items metadata (not: data)""" + repository = self.open_repository(args.archive) + manifest, key = Manifest.load(repository) + archive = Archive(repository, key, manifest, args.archive.archive) + for i, item_id in enumerate(archive.metadata[b'items']): + data = key.decrypt(item_id, repository.get(item_id)) + filename = '%06d_%s.items' %(i, hexlify(item_id).decode('ascii')) + print('Dumping', filename) + with open(filename, 'wb') as fd: + fd.write(data) + print('Done.') + return EXIT_SUCCESS + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -985,6 +999,18 @@ class Archiver: subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices)) subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') + + debug_dump_archive_items_epilog = textwrap.dedent(""" + This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. + """) + subparser = subparsers.add_parser('debug-dump-archive-items', parents=[common_parser], + description=self.do_debug_dump_archive_items.__doc__, + epilog=debug_dump_archive_items_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_dump_archive_items) + subparser.add_argument('archive', metavar='ARCHIVE', + type=location_validator(archive=True), + help='archive to dump') return parser def parse_args(self, args=None): From 47813f6f6a7d747a9b992e697b712ce90b01e11c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 23:45:49 +0100 Subject: [PATCH 029/321] archiver checker: better error logging, give chunk_id and sequence numbers can be used together with borg debug-dump-archive-items. --- borg/archive.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 2333a102..8e850873 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -788,21 +788,33 @@ class ArchiveChecker: _state += 1 return _state + def report(msg, chunk_id, chunk_no): + cid = hexlify(chunk_id).decode('ascii') + msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items + self.report_progress(msg, error=True) + + i = 0 for state, items in groupby(archive[b'items'], missing_chunk_detector): items = list(items) if state % 2: - self.report_progress('Archive metadata damage detected', error=True) + for chunk_id in items: + report('item metadata chunk missing', chunk_id, i) + i += 1 continue if state > 0: unpacker.resync() for chunk_id, cdata in zip(items, repository.get_many(items)): unpacker.feed(self.key.decrypt(chunk_id, cdata)) - for item in unpacker: - if not isinstance(item, dict): - self.report_progress('Did not get expected metadata dict - archive corrupted!', - error=True) - continue - yield item + try: + for item in unpacker: + if isinstance(item, dict): + yield item + else: + report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i) + except Exception: + report('Exception while unpacking item metadata', chunk_id, i) + raise + i += 1 repository = cache_if_remote(self.repository) if archive is None: From 39b5734b31cfc2733891ca20938a2e86f8edeab6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 4 Nov 2015 01:05:21 +0100 Subject: [PATCH 030/321] debugging helper: borg debug-delete-obj --- borg/archiver.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 145ca1e6..9ad291aa 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,7 +17,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ - get_cache_dir, get_keys_dir, prune_within, prune_split, \ + get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR @@ -514,6 +514,28 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_delete_obj(self, args): + """delete the objects with the given IDs from the repo""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + modified = False + for hex_id in args.ids: + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + repository.delete(id) + modified = True + print("object %s deleted." % hex_id) + except repository.ObjectNotFound: + print("object %s not found." % hex_id) + if modified: + repository.commit() + print('Done.') + return EXIT_SUCCESS + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -1011,6 +1033,20 @@ class Archiver: subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to dump') + + debug_delete_obj_epilog = textwrap.dedent(""" + This command deletes objects from the repository. + """) + subparser = subparsers.add_parser('debug-delete-obj', parents=[common_parser], + description=self.do_debug_delete_obj.__doc__, + epilog=debug_delete_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_delete_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to delete from the repo') return parser def parse_args(self, args=None): From f6244f006eec227cb25df21ef551ffd2ea69baa3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 4 Nov 2015 01:51:09 +0100 Subject: [PATCH 031/321] docs: warn about not running out of space --- docs/quickstart.rst | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 19ac429b..6890ab94 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -8,6 +8,29 @@ This chapter will get you started with |project_name|. The first section presents a simple step by step example that uses |project_name| to backup data. The next section continues by showing how backups can be automated. +Important note about free space +------------------------------- + +Before you start creating backups, please make sure that there is **always** +a good amount of free space on the filesystem that has your backup repository +(and also on ~/.cache). It is hard to tell how much, maybe 1-5%. + +If you run out of disk space, it can be hard or impossible to free space, +because |project_name| needs free space to operate - even to delete backup +archives. + +You can use some monitoring process or just include the free space information +in your backup log files (you check them regularly anyway, right?). + +Also helpful: + +- create a big file as a "space reserve", that you can delete to free space +- if you use LVM: use a LV + a filesystem that you can resize later and have + some unallocated PEs you can add to the LV. +- consider using quotas +- use `prune` regularly + + A step by step example ---------------------- From 4c6be00d65321651b562d0e82d43892de6c7446e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 14:58:12 +0100 Subject: [PATCH 032/321] remove some superfluous / duplicate log messages --- borg/cache.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borg/cache.py b/borg/cache.py index b31f68a4..38ef4208 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -43,7 +43,6 @@ class Cache: self.manifest = manifest self.path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii')) self.do_files = do_files - logger.info('initializing cache') # Warn user before sending data to a never seen before unencrypted repository if not os.path.exists(self.path): if warn_if_unencrypted and isinstance(key, PlaintextKey): @@ -71,7 +70,6 @@ class Cache: # Make sure an encrypted repository has not been swapped for an unencrypted repository if self.key_type is not None and self.key_type != str(key.TYPE): raise self.EncryptionMethodMismatch() - logger.info('synchronizing cache') self.sync() self.commit() From a4bb85970d2728690e4b4a5e6b5a1590cf8f78cb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 15:01:37 +0100 Subject: [PATCH 033/321] do not mention the deprecated passphrase mode --- borg/key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/key.py b/borg/key.py index bbe8bb58..78bfa8b7 100644 --- a/borg/key.py +++ b/borg/key.py @@ -90,7 +90,7 @@ class PlaintextKey(KeyBase): @classmethod def create(cls, repository, args): - logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile|passphrase" to enable encryption.') + logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.') return cls(repository) @classmethod From 7da730a8d730f89be79b860366118788271a17d7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 15:45:49 +0100 Subject: [PATCH 034/321] borg list --prefix=thishostname- REPO, fixes #205 lists only archives with names starting with the given prefix. --- borg/archiver.py | 5 +++++ borg/testsuite/archiver.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index e18ded6c..8432a78d 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -414,6 +414,8 @@ class Archiver: remove_surrogates(item[b'path']), extra)) else: for archive_info in manifest.list_archive_infos(sort_by='ts'): + if args.prefix and not archive_info.name.startswith(args.prefix): + continue print(format_archive(archive_info)) return self.exit_code @@ -835,9 +837,12 @@ class Archiver: subparser.add_argument('--short', dest='short', action='store_true', default=False, help='only print file/directory names, nothing else') + subparser.add_argument('-p', '--prefix', dest='prefix', type=str, + help='only consider archive names starting with this prefix') subparser.add_argument('src', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository/archive to list contents of') + mount_epilog = textwrap.dedent(""" This command mounts an archive as a FUSE filesystem. This can be useful for browsing an archive or restoring individual files. Unless the ``--foreground`` diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 61d954e6..780a976e 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -675,6 +675,16 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in('bar-2015-08-12-10:00', output) self.assert_in('bar-2015-08-12-20:00', output) + def test_list_prefix(self): + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test-1', src_dir) + self.cmd('create', self.repository_location + '::something-else-than-test-1', src_dir) + self.cmd('create', self.repository_location + '::test-2', src_dir) + output = self.cmd('list', '--prefix=test-', self.repository_location) + self.assert_in('test-1', output) + self.assert_in('test-2', output) + self.assert_not_in('something-else', output) + def test_usage(self): if self.FORK_DEFAULT: self.cmd(exit_code=0) From bc6b92f13036ee8d20f6937162a46b3f34751794 Mon Sep 17 00:00:00 2001 From: Jan Date: Fri, 6 Nov 2015 16:22:05 +0100 Subject: [PATCH 035/321] Fix for link to "The Borg collective" Fixed the link to "The Borg collective". --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 63cd7dc8..36408513 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ Notes Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_". -.. _The Borg collective: https://borgbackup.readthedocs.org/authors.html +.. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html Differences between Attic and Borg ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 998670576077ad62645a0fc8e984a0147aaed6d3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 16:51:39 +0100 Subject: [PATCH 036/321] add some tests for the debug commands --- borg/testsuite/archiver.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 61d954e6..f9c43835 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -767,6 +767,31 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_aes_counter_uniqueness_passphrase(self): self.verify_aes_counter_uniqueness('passphrase') + def test_debug_dump_archive_items(self): + self.create_test_files() + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir('output'): + output = self.cmd('debug-dump-archive-items', self.repository_location + '::test') + output_dir = sorted(os.listdir('output')) + assert len(output_dir) > 0 and output_dir[0].startswith('000000_') + assert 'Done.' in output + + def test_debug_delete_obj(self): + self.cmd('init', self.repository_location) + repository = Repository(self.repository_location) + data = b'some data' + h = sha256(data) + key, hexkey = h.digest(), h.hexdigest() + repository.put(key, data) + repository.commit() + output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') + assert "is invalid" in output + output = self.cmd('debug-delete-obj', self.repository_location, hexkey) + assert "deleted" in output + output = self.cmd('debug-delete-obj', self.repository_location, hexkey) + assert "not found" in output + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): @@ -875,3 +900,7 @@ class RemoteArchiverTestCase(ArchiverTestCase): @unittest.skip('deadlock issues') def test_fuse_mount_archive(self): pass + + @unittest.skip('only works locally') + def test_debug_delete_obj(self): + pass From a2fc479da38a4e0e49b2cd31ad9ca400cc265188 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 17:31:05 +0100 Subject: [PATCH 037/321] debug-put-obj command --- borg/archiver.py | 28 ++++++++++++++++++++++++++++ borg/testsuite/archiver.py | 17 ++++++++--------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9ad291aa..fe46a20f 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -3,6 +3,7 @@ from .support import argparse # see support/__init__.py docstring from binascii import hexlify from datetime import datetime +from hashlib import sha256 from operator import attrgetter import functools import inspect @@ -514,6 +515,19 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_put_obj(self, args): + """put file(s) contents into the repository""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + for path in args.paths: + with open(path, "rb") as f: + data = f.read() + h = sha256(data) # XXX hardcoded + repository.put(h.digest(), data) + print("object %s put." % h.hexdigest()) + repository.commit() + return EXIT_SUCCESS + def do_debug_delete_obj(self, args): """delete the objects with the given IDs from the repo""" repository = self.open_repository(args.repository) @@ -1034,6 +1048,20 @@ class Archiver: type=location_validator(archive=True), help='archive to dump') + debug_put_obj_epilog = textwrap.dedent(""" + This command puts objects into the repository. + """) + subparser = subparsers.add_parser('debug-put-obj', parents=[common_parser], + description=self.do_debug_put_obj.__doc__, + epilog=debug_put_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_put_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, + help='file(s) to read and create object(s) from') + debug_delete_obj_epilog = textwrap.dedent(""" This command deletes objects from the repository. """) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index f9c43835..717c81aa 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -777,20 +777,19 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert len(output_dir) > 0 and output_dir[0].startswith('000000_') assert 'Done.' in output - def test_debug_delete_obj(self): + def test_debug_put_delete_obj(self): self.cmd('init', self.repository_location) - repository = Repository(self.repository_location) data = b'some data' - h = sha256(data) - key, hexkey = h.digest(), h.hexdigest() - repository.put(key, data) - repository.commit() - output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') - assert "is invalid" in output + hexkey = sha256(data).hexdigest() + self.create_regular_file('file', contents=data) + output = self.cmd('debug-put-obj', self.repository_location, 'input/file') + assert hexkey in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "deleted" in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "not found" in output + output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') + assert "is invalid" in output @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') @@ -902,5 +901,5 @@ class RemoteArchiverTestCase(ArchiverTestCase): pass @unittest.skip('only works locally') - def test_debug_delete_obj(self): + def test_debug_put_delete_obj(self): pass From 37c8aa2d428b2a7170fc894512ffed124318c2cf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 17:45:30 +0100 Subject: [PATCH 038/321] debug-get-obj command --- borg/archiver.py | 36 ++++++++++++++++++++++++++++++++++++ borg/testsuite/archiver.py | 9 +++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index fe46a20f..1af8e46e 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -515,6 +515,26 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_get_obj(self, args): + """get object contents from the repository and write it into file""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + hex_id = args.id + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + data =repository.get(id) + except repository.ObjectNotFound: + print("object %s not found." % hex_id) + else: + with open(args.path, "wb") as f: + f.write(data) + print("object %s fetched." % hex_id) + return EXIT_SUCCESS + def do_debug_put_obj(self, args): """put file(s) contents into the repository""" repository = self.open_repository(args.repository) @@ -1048,6 +1068,22 @@ class Archiver: type=location_validator(archive=True), help='archive to dump') + debug_get_obj_epilog = textwrap.dedent(""" + This command gets an object from the repository. + """) + subparser = subparsers.add_parser('debug-get-obj', parents=[common_parser], + description=self.do_debug_get_obj.__doc__, + epilog=debug_get_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_get_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('id', metavar='ID', type=str, + help='hex object ID to get from the repo') + subparser.add_argument('path', metavar='PATH', type=str, + help='file to write object data into') + debug_put_obj_epilog = textwrap.dedent(""" This command puts objects into the repository. """) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 717c81aa..8e7cbf6e 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -777,13 +777,18 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert len(output_dir) > 0 and output_dir[0].startswith('000000_') assert 'Done.' in output - def test_debug_put_delete_obj(self): + def test_debug_put_get_delete_obj(self): self.cmd('init', self.repository_location) data = b'some data' hexkey = sha256(data).hexdigest() self.create_regular_file('file', contents=data) output = self.cmd('debug-put-obj', self.repository_location, 'input/file') assert hexkey in output + output = self.cmd('debug-get-obj', self.repository_location, hexkey, 'output/file') + assert hexkey in output + with open('output/file', 'rb') as f: + data_read = f.read() + assert data == data_read output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "deleted" in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) @@ -901,5 +906,5 @@ class RemoteArchiverTestCase(ArchiverTestCase): pass @unittest.skip('only works locally') - def test_debug_put_delete_obj(self): + def test_debug_put_get_delete_obj(self): pass From 6c1b765741f73ccd8cc99d82e2000ae827de12e3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 18:22:30 +0100 Subject: [PATCH 039/321] docs: group general stuff under 1 headline, fix another headline level --- docs/usage.rst | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 32345c3b..699666b0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -8,15 +8,18 @@ Usage a number of arguments and options. The following sections will describe each command in detail. +General +------- + Quiet by default ----------------- +~~~~~~~~~~~~~~~~ Like most UNIX commands |project_name| is quiet by default but the ``-v`` or ``--verbose`` option can be used to get the program to output more status messages as it is processing. Return codes ------------- +~~~~~~~~~~~~ |project_name| can exit with the following return codes (rc): @@ -33,7 +36,7 @@ The return code is also logged at the indicated level as the last log entry. Environment Variables ---------------------- +~~~~~~~~~~~~~~~~~~~~~ |project_name| uses some environment variables for automation: @@ -85,7 +88,7 @@ Please note: Resource Usage --------------- +~~~~~~~~~~~~~~ |project_name| might use a lot of resources depending on the size of the data set it is dealing with. @@ -131,7 +134,7 @@ In case you are interested in more details, please read the internals documentat Units ------ +~~~~~ To display quantities, |project_name| takes care of respecting the usual conventions of scale. Disk sizes are displayed in `decimal @@ -476,7 +479,7 @@ maybe directly into an existing device file of your choice or indirectly via ``dd``). Example -~~~~~~~ ++++++++ Imagine you have made some snapshots of logical volumes (LVs) you want to backup. From b1eb784bd1a569f0fcb736f4fed635fe3c96a608 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 18:31:06 +0100 Subject: [PATCH 040/321] docs: add section about debug commands --- docs/usage.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index 699666b0..46ecd4bf 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -402,6 +402,16 @@ Miscellaneous Help .. include:: usage/help.rst.inc +Debug Commands +-------------- +There are some more commands (all starting with "debug-") wich are are all +**not intended for normal use** and **potentially very dangerous** if used incorrectly. + +They exist to improve debugging capabilities without direct system access, e.g. +in case you ever run into some severe malfunction. Use them only if you know +what you are doing or if a trusted |project_name| developer tells you what to do. + + Additional Notes ---------------- From c01efa46663f634543e09f81c0a1cf8cdd107654 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 19:37:39 +0100 Subject: [PATCH 041/321] repository: refactor some duplicate code --- borg/repository.py | 84 +++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 49 deletions(-) diff --git a/borg/repository.py b/borg/repository.py index e1651be2..048fa032 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -243,36 +243,44 @@ class Repository: continue if segment > segments_transaction_id: break - # code duplication below?? vvv (see similar code in check()) - self.segments[segment] = 0 - for tag, key, offset in self.io.iter_objects(segment): - if tag == TAG_PUT: - try: - s, _ = self.index[key] - self.compact.add(s) - self.segments[s] -= 1 - except KeyError: - pass - self.index[key] = segment, offset - self.segments[segment] += 1 - elif tag == TAG_DELETE: - try: - s, _ = self.index.pop(key) - self.segments[s] -= 1 - self.compact.add(s) - except KeyError: - pass - self.compact.add(segment) - elif tag == TAG_COMMIT: - continue - else: - raise self.CheckNeeded(self.path) - if self.segments[segment] == 0: - self.compact.add(segment) + objects = self.io.iter_objects(segment) + self._update_index(segment, objects) self.write_index() finally: self.rollback() + def _update_index(self, segment, objects, report=None): + """some code shared between replay_segments and check""" + self.segments[segment] = 0 + for tag, key, offset in objects: + if tag == TAG_PUT: + try: + s, _ = self.index[key] + self.compact.add(s) + self.segments[s] -= 1 + except KeyError: + pass + self.index[key] = segment, offset + self.segments[segment] += 1 + elif tag == TAG_DELETE: + try: + s, _ = self.index.pop(key) + self.segments[s] -= 1 + self.compact.add(s) + except KeyError: + pass + self.compact.add(segment) + elif tag == TAG_COMMIT: + continue + else: + msg = 'Unexpected tag {} in segment {}'.format(tag, segment) + if report is None: + raise self.CheckNeeded(msg) + else: + report(msg) + if self.segments[segment] == 0: + self.compact.add(segment) + def check(self, repair=False): """Check repository consistency @@ -312,29 +320,7 @@ class Repository: if repair: self.io.recover_segment(segment, filename) objects = list(self.io.iter_objects(segment)) - self.segments[segment] = 0 - for tag, key, offset in objects: - if tag == TAG_PUT: - try: - s, _ = self.index[key] - self.compact.add(s) - self.segments[s] -= 1 - except KeyError: - pass - self.index[key] = segment, offset - self.segments[segment] += 1 - elif tag == TAG_DELETE: - try: - s, _ = self.index.pop(key) - self.segments[s] -= 1 - self.compact.add(s) - except KeyError: - pass - self.compact.add(segment) - elif tag == TAG_COMMIT: - continue - else: - report_error('Unexpected tag {} in segment {}'.format(tag, segment)) + self._update_index(segment, objects, report_error) # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found if repair and segments_transaction_id is None: From 47fec587d4ad3032f79c266fc3abcc1c839c366e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 21:33:43 +0100 Subject: [PATCH 042/321] updated CHANGES --- docs/changes.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 620e6614..154f2348 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -12,17 +12,29 @@ New features: - extract: support atime additionally to mtime - FUSE: support ctime and atime additionally to mtime - support borg --version +- emit a warning if we have a slow msgpack installed +- borg list --prefix=thishostname- REPO, fixes #205 +- Debug commands (do not use except if you know what you do: debug-get-obj, + debug-put-obj, debug-delete-obj, debug-dump-archive-items. +- add --show-rc option enable "terminating with X status, rc N" output, fixes #351 Bug fixes: - setup.py: fix bug related to BORG_LZ4_PREFIX processing - borg mount: fix unlocking of repository at umount time, fixes #331 +- add a test to find disk-full issues, #327 +- fix "check" for repos that have incomplete chunks, fixes #364 - fix reading files without touching their atime, #334 - non-ascii ACL fixes for Linux, FreeBSD and OS X, #277 - fix acl_use_local_uid_gid() and add a test for it, attic #359 - borg upgrade: do not upgrade repositories in place by default, #299 - fix cascading failure with the index conversion code, #269 - borg check: implement 'cmdline' archive metadata value decoding, #311 +- fix RobustUnpacker, it missed some metadata keys (new atime and ctime keys + were missing, but also bsdflags). add check for unknown metadata keys. +- create from stdin: also save atime, ctime (cosmetic) +- use default_notty=False for confirmations, fixes #345 +- vagrant: fix msgpack installation on centos, fixes #342 Other changes: @@ -39,6 +51,14 @@ Other changes: maybe fixes #309 - benchmarks: test create, extract, list, delete, info, check, help, fixes #146 - benchmarks: test with both the binary and the python code +- move away from RawConfigParser to ConfigParser +- archive checker: better error logging, give chunk_id and sequence numbers + (can be used together with borg debug-dump-archive-items). +- do not mention the deprecated passphrase mode +- emit a deprecation warning for --compression N (giving a just a number) +- misc .coverragerc fixes (and coverage measurment improvements), fixes #319 +- refactor confirmation code, reduce code duplication, add tests +- prettier error messages, fixes #307, #57 - tests: - travis: also run tests on Python 3.5 @@ -47,6 +67,7 @@ Other changes: - vagrant: tests: announce whether fakeroot is used or not - vagrant: add vagrant user to fuse group for debianoid systems also - vagrant: llfuse install on darwin needs pkgconfig installed + - vagrant: use pyinstaller from develop branch, fixes #336 - archiver tests: test with both the binary and the python code, fixes #215 - docs: @@ -64,7 +85,11 @@ Other changes: - remove api docs (too much breakage on rtd) - borgbackup install + basics presentation (asciinema) - describe the current style guide in documentation - + - add section about debug commands + - warn about not running out of space + - add example for rename + - improve chunker params docs, fixes #362 + Version 0.27.0 -------------- From c34ef375160f0669c2fd101c906df4347e9d6637 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 22:04:37 +0100 Subject: [PATCH 043/321] CHANGES: some fixes, add compatibility notes --- docs/changes.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 154f2348..042bef39 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -4,10 +4,14 @@ Changelog Version 0.28.0 -------------- +Compatibility notes: +- changed return codes (exit codes), see docs. in short: + old: 0 = ok, 1 = error. now: 0 = ok, 1 = warning, 2 = error + New features: - refactor return codes (exit codes), fixes #61 -- give a final status into the log output, including exit code, fixes #58 +- add --show-rc option enable "terminating with X status, rc N" output, fixes 58, #351 - borg create backups atime and ctime additionally to mtime, fixes #317 - extract: support atime additionally to mtime - FUSE: support ctime and atime additionally to mtime @@ -16,14 +20,12 @@ New features: - borg list --prefix=thishostname- REPO, fixes #205 - Debug commands (do not use except if you know what you do: debug-get-obj, debug-put-obj, debug-delete-obj, debug-dump-archive-items. -- add --show-rc option enable "terminating with X status, rc N" output, fixes #351 Bug fixes: - setup.py: fix bug related to BORG_LZ4_PREFIX processing -- borg mount: fix unlocking of repository at umount time, fixes #331 -- add a test to find disk-full issues, #327 - fix "check" for repos that have incomplete chunks, fixes #364 +- borg mount: fix unlocking of repository at umount time, fixes #331 - fix reading files without touching their atime, #334 - non-ascii ACL fixes for Linux, FreeBSD and OS X, #277 - fix acl_use_local_uid_gid() and add a test for it, attic #359 @@ -38,9 +40,8 @@ Bug fixes: Other changes: -- improve file size displays -- convert to more flexible size formatters -- explicitely commit to the units standard, #289 +- improve file size displays, more flexible size formatters +- explicitly commit to the units standard, #289 - archiver: add E status (means that an error occured when processing this (single) item - do binary releases via "github releases", closes #214 @@ -49,18 +50,17 @@ Other changes: - show progress display if on a tty, output more progress information, #303 - factor out status output so it is consistent, fix surrogates removal, maybe fixes #309 -- benchmarks: test create, extract, list, delete, info, check, help, fixes #146 -- benchmarks: test with both the binary and the python code - move away from RawConfigParser to ConfigParser - archive checker: better error logging, give chunk_id and sequence numbers (can be used together with borg debug-dump-archive-items). - do not mention the deprecated passphrase mode - emit a deprecation warning for --compression N (giving a just a number) -- misc .coverragerc fixes (and coverage measurment improvements), fixes #319 +- misc .coverragerc fixes (and coverage measurement improvements), fixes #319 - refactor confirmation code, reduce code duplication, add tests - prettier error messages, fixes #307, #57 - tests: + - add a test to find disk-full issues, #327 - travis: also run tests on Python 3.5 - travis: use tox -r so it rebuilds the tox environments - test the generated pyinstaller-based binary by archiver unit tests, #215 @@ -68,6 +68,8 @@ Other changes: - vagrant: add vagrant user to fuse group for debianoid systems also - vagrant: llfuse install on darwin needs pkgconfig installed - vagrant: use pyinstaller from develop branch, fixes #336 + - benchmarks: test create, extract, list, delete, info, check, help, fixes #146 + - benchmarks: test with both the binary and the python code - archiver tests: test with both the binary and the python code, fixes #215 - docs: From eb3bc1023b26b4bad172cea5dc4cbdb776e363bd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 00:11:36 +0100 Subject: [PATCH 044/321] make basic test more robust counting lines is a bad idea. just one unrelated output line and the test fails. thus we rather check if what we expect is in the output. --- borg/testsuite/archiver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index a70cd7ea..8ec0297f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -304,7 +304,9 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('create', '--stats', self.repository_location + '::test.2', 'input') with changedir('output'): self.cmd('extract', self.repository_location + '::test') - self.assert_equal(len(self.cmd('list', self.repository_location).splitlines()), 2) + list_output = self.cmd('list', self.repository_location) + self.assert_in('test ', list_output) + self.assert_in('test.2 ', list_output) expected = [ 'input', 'input/bdev', From 8cc726a1076af5e638f892e0c250f8f43f61b334 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 00:30:02 +0100 Subject: [PATCH 045/321] make basic test more robust, try 2 check if we have all expected files in list output (but ignore extra lines, like "can't load libfakeroot" or so). --- borg/testsuite/archiver.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8ec0297f..ca24807d 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -304,9 +304,9 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('create', '--stats', self.repository_location + '::test.2', 'input') with changedir('output'): self.cmd('extract', self.repository_location + '::test') - list_output = self.cmd('list', self.repository_location) - self.assert_in('test ', list_output) - self.assert_in('test.2 ', list_output) + list_output = self.cmd('list', '--short', self.repository_location) + self.assert_in('test', list_output) + self.assert_in('test.2', list_output) expected = [ 'input', 'input/bdev', @@ -328,7 +328,9 @@ class ArchiverTestCase(ArchiverTestCaseBase): # remove the file we did not backup, so input and output become equal expected.remove('input/flagfile') # this file is UF_NODUMP os.remove(os.path.join('input', 'flagfile')) - self.assert_equal(self.cmd('list', '--short', self.repository_location + '::test').splitlines(), expected) + list_output = self.cmd('list', '--short', self.repository_location + '::test') + for name in expected: + self.assert_in(name, list_output) self.assert_dirs_equal('input', 'output/input') info_output = self.cmd('info', self.repository_location + '::test') item_count = 3 if has_lchflags else 4 # one file is UF_NODUMP From 303bed9d122735a3dcf74db6a3cd2c09b5979fae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 15:17:40 +0100 Subject: [PATCH 046/321] docs: minor development docs update --- docs/development.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 75ec53df..a8d664f6 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -83,7 +83,7 @@ main repository. Using Vagrant ------------- -We use Vagrant for the automated creation of testing environment and borgbackup +We use Vagrant for the automated creation of testing environments and borgbackup standalone binaries for various platforms. For better security, there is no automatic sync in the VM to host direction. @@ -100,7 +100,7 @@ Usage:: To shut down and destroy the VM: vagrant destroy OS To copy files from the VM (in this case, the generated binary): - vagrant scp OS:/vagrant/borg/borg/dist/borg . + vagrant scp OS:/vagrant/borg/borg.exe . Creating standalone binaries From c9090fc3c2c9a827962699d68b33884ba06423df Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 15:37:05 +0100 Subject: [PATCH 047/321] helper: minor style fix --- borg/helpers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index c371c7b3..d1eb5d92 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -650,13 +650,14 @@ class Location: return False def __str__(self): - items = [] - items.append('proto=%r' % self.proto) - items.append('user=%r' % self.user) - items.append('host=%r' % self.host) - items.append('port=%r' % self.port) - items.append('path=%r' % self.path) - items.append('archive=%r' % self.archive) + items = [ + 'proto=%r' % self.proto, + 'user=%r' % self.user, + 'host=%r' % self.host, + 'port=%r' % self.port, + 'path=%r' % self.path, + 'archive=%r' % self.archive, + ] return ', '.join(items) def to_key_filename(self): From 3d2a1da98330d23558303dfccb0598f00c5fb553 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 16:13:24 +0100 Subject: [PATCH 048/321] use latest pytest-benchmark --- requirements.d/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 5ec1ed14..28195f54 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -2,5 +2,5 @@ tox mock pytest pytest-cov<2.0.0 -pytest-benchmark==3.0.0b1 +pytest-benchmark==3.0.0rc1 Cython From 244303ac668471ebb638f7278645a381267d239e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 7 Nov 2015 19:55:17 +0100 Subject: [PATCH 049/321] add ACL keys the RobustUnpacker must know about --- borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index e9a9a7d9..524c2a5a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -595,7 +595,7 @@ Number of files: {0.stats.nfiles}'''.format(self) # this set must be kept complete, otherwise the RobustUnpacker might malfunction: ITEM_KEYS = set([b'path', b'source', b'rdev', b'chunks', b'mode', b'user', b'group', b'uid', b'gid', b'mtime', b'atime', b'ctime', - b'xattrs', b'bsdflags', + b'xattrs', b'bsdflags', b'acl_nfs4', b'acl_access', b'acl_default', b'acl_extended', ]) From ba67b96434253d9f0585d586f56af0b4c70764b0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 00:57:02 +0100 Subject: [PATCH 050/321] have a helpful warning message about how to fix wrong locale setup, fixes #382 --- borg/archiver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index acb705ff..88a42331 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -261,6 +261,8 @@ class Archiver: # be restrictive when restoring files, restore permissions later if sys.getfilesystemencoding() == 'ascii': logger.warning('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.') + if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): + logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.archive.archive, From 5595d56ecf26678e36dca4ec6e7c128c7ab7325d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 01:05:55 +0100 Subject: [PATCH 051/321] deal with unicode errors for symlinks in same way as for regular files, fixes #382 --- borg/archive.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index 524c2a5a..7edd4162 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -342,7 +342,10 @@ Number of files: {0.stats.nfiles}'''.format(self) source = item[b'source'] if os.path.exists(path): os.unlink(path) - os.symlink(source, path) + try: + os.symlink(source, path) + except UnicodeEncodeError: + raise self.IncompatibleFilesystemEncodingError(source, sys.getfilesystemencoding()) self.restore_attrs(path, item, symlink=True) elif stat.S_ISFIFO(mode): if not os.path.exists(os.path.dirname(path)): From cd40ec280d931e75a1637031395b6dc27cb2cf7f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 01:29:53 +0100 Subject: [PATCH 052/321] update CHANGES --- docs/changes.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 042bef39..f9057f92 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -37,6 +37,9 @@ Bug fixes: - create from stdin: also save atime, ctime (cosmetic) - use default_notty=False for confirmations, fixes #345 - vagrant: fix msgpack installation on centos, fixes #342 +- deal with unicode errors for symlinks in same way as for regular files and + have a helpful warning message about how to fix wrong locale setup, fixes #382 +- add ACL keys the RobustUnpacker must know about Other changes: @@ -71,6 +74,7 @@ Other changes: - benchmarks: test create, extract, list, delete, info, check, help, fixes #146 - benchmarks: test with both the binary and the python code - archiver tests: test with both the binary and the python code, fixes #215 + - make basic test more robust - docs: - moved docs to borgbackup.readthedocs.org, #155 @@ -91,6 +95,7 @@ Other changes: - warn about not running out of space - add example for rename - improve chunker params docs, fixes #362 + - minor development docs update Version 0.27.0 From 007572594f2d966f50aab5ecb5b0824600e3f6e2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 02:00:51 +0100 Subject: [PATCH 053/321] try to fix build on readthedocs --- borg/helpers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index d1eb5d92..1706358d 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -42,8 +42,7 @@ if have_cython(): from . import chunker from . import crypto import msgpack - -import msgpack.fallback + import msgpack.fallback # return codes returned by borg command From f48a5ae6a74a30c0294731b0dda00266e5ee8dc2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 02:17:37 +0100 Subject: [PATCH 054/321] fix formatting issue in changes.rst --- docs/changes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changes.rst b/docs/changes.rst index f9057f92..2a8ed409 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,6 +5,7 @@ Version 0.28.0 -------------- Compatibility notes: + - changed return codes (exit codes), see docs. in short: old: 0 = ok, 1 = error. now: 0 = ok, 1 = warning, 2 = error From e6911b2f254bef8a3968a48dcdc59c8e2931933a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ketelaars?= Date: Sun, 8 Nov 2015 14:24:58 +0100 Subject: [PATCH 055/321] Avoid using msgpack.packb at import time. move item_keys = ... to __init__ (self.item_keys = ...) Solution from/discussed with TW. --- borg/archive.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 7edd4162..008909a2 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -605,10 +605,9 @@ ITEM_KEYS = set([b'path', b'source', b'rdev', b'chunks', class RobustUnpacker: """A restartable/robust version of the streaming msgpack unpacker """ - item_keys = [msgpack.packb(name) for name in ITEM_KEYS] - def __init__(self, validator): super().__init__() + self.item_keys = [msgpack.packb(name) for name in ITEM_KEYS] self.validator = validator self._buffered_data = [] self._resync = False From 3238b8ea76013a3d8a3b8d6089a31d34ab55ddb7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 16:16:43 +0100 Subject: [PATCH 056/321] do not try to build api / usage docs for production install it requires "mock" (and later also sphinx) and we do not (want to) have that in setup_requires. I removed the build.sub_commands.append(...) because that made setup.py install run build_api and build_usage automatically for production installs. the comment said it does not work on readthedocs anyway. build_api and build_usage can still be run manually. --- setup.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 284fddd0..06795938 100644 --- a/setup.py +++ b/setup.py @@ -202,11 +202,6 @@ API Documentation :undoc-members: """ % mod) -# (function, predicate), see http://docs.python.org/2/distutils/apiref.html#distutils.cmd.Command.sub_commands -# seems like this doesn't work on RTD, see below for build_py hack. -build.sub_commands.append(('build_api', None)) -build.sub_commands.append(('build_usage', None)) - class build_py_custom(build_py): """override build_py to also build our stuff @@ -227,8 +222,13 @@ class build_py_custom(build_py): super().run() self.announce('calling custom build steps', level=log.INFO) self.run_command('build_ext') - self.run_command('build_api') - self.run_command('build_usage') + if on_rtd: + # only build these files if running on readthedocs, but not + # for a normal production install. It requires "mock" and we + # do not have that as a build dependency. Also, for really + # building the docs, it would also require sphinx, etc. + self.run_command('build_api') + self.run_command('build_usage') cmdclass = { From 4a2e4ec683c23d3ac242467cbd4b8983252042f8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 8 Nov 2015 17:10:18 +0100 Subject: [PATCH 057/321] update CHANGES --- docs/changes.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 2a8ed409..fea8bfd2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,21 @@ Changelog ========= +Version 0.28.1 +-------------- + +Bug fixes: + +- do not try to build api / usage docs for production install, + fixes unexpected "mock" build dependency, #384 + +Other changes: + +- avoid using msgpack.packb at import time +- fix formatting issue in changes.rst +- fix build on readthedocs + + Version 0.28.0 -------------- From a6a8a4ebd9af652cd3eaf63362ee0d8d8d4bff13 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 9 Nov 2015 03:41:06 +0100 Subject: [PATCH 058/321] Implement --exclude-if-present Add a new --exclude-if-present command-line flag to ``borg create``. If specified, directories containing the specified tag file will be excluded from the backup. The flag can be repeated to ignore more than a single tag file, irregardless of the contents. This is taken from a attic PR (and adapted for borg): commit 3462a9ca90388dc5d8b4fa4218a32769676b3623 Author: Yuri D'Elia Date: Sun Dec 7 19:15:17 2014 +0100 --- borg/archiver.py | 19 +++++++++++-------- borg/helpers.py | 16 +++++++++++++++- borg/testsuite/archiver.py | 11 +++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 88a42331..4e468205 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -20,7 +20,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ + dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .logger import create_logger, setup_logging logger = create_logger() @@ -166,8 +166,8 @@ class Archiver: continue else: restrict_dev = None - self._process(archive, cache, args.excludes, args.exclude_caches, skip_inodes, path, restrict_dev, - read_special=args.read_special, dry_run=dry_run) + self._process(archive, cache, args.excludes, args.exclude_caches, args.exclude_if_present, + skip_inodes, path, restrict_dev, read_special=args.read_special, dry_run=dry_run) if not dry_run: archive.save(timestamp=args.timestamp) if args.progress: @@ -182,8 +182,8 @@ class Archiver: print('-' * 78) return self.exit_code - def _process(self, archive, cache, excludes, exclude_caches, skip_inodes, path, restrict_dev, - read_special=False, dry_run=False): + def _process(self, archive, cache, excludes, exclude_caches, exclude_if_present, + skip_inodes, path, restrict_dev, read_special=False, dry_run=False): if exclude_path(path, excludes): return try: @@ -209,7 +209,7 @@ class Archiver: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): - if exclude_caches and is_cachedir(path): + if dir_is_tagged(path, exclude_caches, exclude_if_present): return if not dry_run: status = archive.process_dir(path, st) @@ -221,8 +221,8 @@ class Archiver: else: for filename in sorted(entries): entry_path = os.path.normpath(os.path.join(path, filename)) - self._process(archive, cache, excludes, exclude_caches, skip_inodes, - entry_path, restrict_dev, read_special=read_special, + self._process(archive, cache, excludes, exclude_caches, exclude_if_present, + skip_inodes, entry_path, restrict_dev, read_special=read_special, dry_run=dry_run) elif stat.S_ISLNK(st.st_mode): if not dry_run: @@ -785,6 +785,9 @@ class Archiver: subparser.add_argument('--exclude-caches', dest='exclude_caches', action='store_true', default=False, help='exclude directories that contain a CACHEDIR.TAG file (http://www.brynosaurus.com/cachedir/spec.html)') + subparser.add_argument('--exclude-if-present', dest='exclude_if_present', + metavar='FILENAME', action='append', type=str, + help='exclude directories that contain the specified file') subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval', type=int, default=300, metavar='SECONDS', help='write checkpoint every SECONDS seconds (Default: 300)') diff --git a/borg/helpers.py b/borg/helpers.py index 1706358d..9fc5c7ba 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -428,7 +428,7 @@ def CompressionSpec(s): raise ValueError -def is_cachedir(path): +def dir_is_cachedir(path): """Determines whether the specified path is a cache directory (and therefore should potentially be excluded from the backup) according to the CACHEDIR.TAG protocol @@ -448,6 +448,20 @@ def is_cachedir(path): return False +def dir_is_tagged(path, exclude_caches, exclude_if_present): + """Determines whether the specified path is excluded by being a cache + directory or containing the user-specified tag file. + """ + if exclude_caches and dir_is_cachedir(path): + return True + if exclude_if_present is not None: + for tag in exclude_if_present: + tag_path = os.path.join(path, tag) + if os.path.isfile(tag_path): + return True + return False + + def format_time(t): """Format datetime suitable for fixed length list output """ diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index ca24807d..02dc5d0a 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -499,6 +499,17 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(sorted(os.listdir('output/input')), ['cache2', 'file1']) self.assert_equal(sorted(os.listdir('output/input/cache2')), ['CACHEDIR.TAG']) + def test_exclude_tagged(self): + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('tagged1/.NOBACKUP') + self.create_regular_file('tagged2/00-NOBACKUP') + self.create_regular_file('tagged3/.NOBACKUP/file2') + self.cmd('create', '--exclude-if-present', '.NOBACKUP', '--exclude-if-present', '00-NOBACKUP', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged3']) + def test_path_normalization(self): self.cmd('init', self.repository_location) self.create_regular_file('dir1/dir2/file', size=1024 * 80) From 7d178e09b0f41fd73c90b6d984db2c8565e4ebff Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 9 Nov 2015 04:08:49 +0100 Subject: [PATCH 059/321] Implement --keep-tag-files to preserve directory roots/tag-files We also add --keep-tag-files to keep in the archive the root directory and the tag/exclusion file in the archive. This is taken from a attic PR (and adapted for borg): commit f61e22cacc90e76e6c8f4b23677eee62c09e97ac Author: Yuri D'Elia Date: Mon Dec 15 12:27:43 2014 +0100 --- borg/archiver.py | 19 ++++++++++++++----- borg/helpers.py | 10 ++++++---- borg/testsuite/archiver.py | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 4e468205..5512bb25 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -167,7 +167,8 @@ class Archiver: else: restrict_dev = None self._process(archive, cache, args.excludes, args.exclude_caches, args.exclude_if_present, - skip_inodes, path, restrict_dev, read_special=args.read_special, dry_run=dry_run) + args.keep_tag_files, skip_inodes, path, restrict_dev, + read_special=args.read_special, dry_run=dry_run) if not dry_run: archive.save(timestamp=args.timestamp) if args.progress: @@ -183,7 +184,8 @@ class Archiver: return self.exit_code def _process(self, archive, cache, excludes, exclude_caches, exclude_if_present, - skip_inodes, path, restrict_dev, read_special=False, dry_run=False): + keep_tag_files, skip_inodes, path, restrict_dev, + read_special=False, dry_run=False): if exclude_path(path, excludes): return try: @@ -209,7 +211,11 @@ class Archiver: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): - if dir_is_tagged(path, exclude_caches, exclude_if_present): + tag_path = dir_is_tagged(path, exclude_caches, exclude_if_present) + if tag_path: + if keep_tag_files: + archive.process_dir(path, st) + archive.process_file(tag_path, st, cache) return if not dry_run: status = archive.process_dir(path, st) @@ -222,8 +228,8 @@ class Archiver: for filename in sorted(entries): entry_path = os.path.normpath(os.path.join(path, filename)) self._process(archive, cache, excludes, exclude_caches, exclude_if_present, - skip_inodes, entry_path, restrict_dev, read_special=read_special, - dry_run=dry_run) + keep_tag_files, skip_inodes, entry_path, restrict_dev, + read_special=read_special, dry_run=dry_run) elif stat.S_ISLNK(st.st_mode): if not dry_run: status = archive.process_symlink(path, st) @@ -788,6 +794,9 @@ class Archiver: subparser.add_argument('--exclude-if-present', dest='exclude_if_present', metavar='FILENAME', action='append', type=str, help='exclude directories that contain the specified file') + subparser.add_argument('--keep-tag-files', dest='keep_tag_files', + action='store_true', default=False, + help='keep tag files of excluded caches/directories') subparser.add_argument('-c', '--checkpoint-interval', dest='checkpoint_interval', type=int, default=300, metavar='SECONDS', help='write checkpoint every SECONDS seconds (Default: 300)') diff --git a/borg/helpers.py b/borg/helpers.py index 9fc5c7ba..84379e75 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -450,16 +450,18 @@ def dir_is_cachedir(path): def dir_is_tagged(path, exclude_caches, exclude_if_present): """Determines whether the specified path is excluded by being a cache - directory or containing the user-specified tag file. + directory or containing the user-specified tag file. Returns the + path of the tag file (either CACHEDIR.TAG or the matching + user-specified file) """ if exclude_caches and dir_is_cachedir(path): - return True + return os.path.join(path, 'CACHEDIR.TAG') if exclude_if_present is not None: for tag in exclude_if_present: tag_path = os.path.join(path, tag) if os.path.isfile(tag_path): - return True - return False + return tag_path + return None def format_time(t): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 02dc5d0a..aaa75833 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -510,6 +510,20 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', self.repository_location + '::test') self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged3']) + def test_exclude_keep_tagged(self): + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('tagged1/.NOBACKUP') + self.create_regular_file('tagged1/file2', size=1024 * 80) + self.create_regular_file('tagged2/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') + self.create_regular_file('tagged2/file3', size=1024 * 80) + self.cmd('create', '--exclude-if-present', '.NOBACKUP', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged1', 'tagged2']) + self.assert_equal(sorted(os.listdir('output/input/tagged1')), ['.NOBACKUP']) + self.assert_equal(sorted(os.listdir('output/input/tagged2')), ['CACHEDIR.TAG']) + def test_path_normalization(self): self.cmd('init', self.repository_location) self.create_regular_file('dir1/dir2/file', size=1024 * 80) From f52fb410a504b1a0fd69d26d7f86ba21d86b0cf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 13 Nov 2015 10:31:56 -0500 Subject: [PATCH 060/321] Revert "use build_py to fix build on RTD", disables docs build This reverts commit 86487d192a9a5ab7ff4eedb92d793485b4c30268. We will instead commit the generated `.rst` usage and API files directly into git. The setup commands remain to generate them when the usage or API changes, but as things are the hoops required to generate those RST files are way too complicated to justify simply build docs. See #384. Conflicts: setup.py --- setup.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/setup.py b/setup.py index 06795938..5c6e98a2 100644 --- a/setup.py +++ b/setup.py @@ -6,8 +6,6 @@ from glob import glob from distutils.command.build import build from distutils.core import Command -from distutils import log -from setuptools.command.build_py import build_py min_python = (3, 2) my_python = sys.version_info @@ -203,39 +201,10 @@ API Documentation """ % mod) -class build_py_custom(build_py): - """override build_py to also build our stuff - - it is unclear why this is necessary, but in some environments - (Readthedocs.org, specifically), the above - ``build.sub_commands.append()`` doesn't seem to have an effect: - our custom build commands seem to be ignored when running - ``setup.py install``. - - This class overrides the ``build_py`` target by forcing it to run - our custom steps as well. - - See also the `bug report on RTD - `_. - """ - def run(self): - super().run() - self.announce('calling custom build steps', level=log.INFO) - self.run_command('build_ext') - if on_rtd: - # only build these files if running on readthedocs, but not - # for a normal production install. It requires "mock" and we - # do not have that as a build dependency. Also, for really - # building the docs, it would also require sphinx, etc. - self.run_command('build_api') - self.run_command('build_usage') - - cmdclass = { 'build_ext': build_ext, 'build_api': build_api, 'build_usage': build_usage, - 'build_py': build_py_custom, 'sdist': Sdist } From f13dd6e579299c3a7997c3722fe8e2450ccea448 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 13 Nov 2015 10:38:50 -0500 Subject: [PATCH 061/321] completely remove have_cython() hack this was making us require mock, which is really a test component and shouldn't be part of the runtime dependencies. furthermore, it was making the imports and the code more brittle: it may have been possible that, through an environment variable, backups could be corrupted because mock libraries would be configured instead of real once, which is a risk we shouldn't be taking. finally, this was used only to build docs, which we will build and commit to git by hand with a fully working borg when relevant. see #384. --- borg/archive.py | 14 +++++--------- borg/archiver.py | 13 ++++++------- borg/cache.py | 5 ++--- borg/fuse.py | 5 ++--- borg/helpers.py | 26 +++++--------------------- borg/key.py | 9 ++++----- borg/remote.py | 5 ++--- borg/repository.py | 5 ++--- docs/usage.rst | 6 ------ setup.py | 5 ----- 10 files changed, 28 insertions(+), 65 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 008909a2..d5b46792 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -18,16 +18,12 @@ import time from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \ - Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, have_cython, \ + Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ st_atime_ns, st_ctime_ns, st_mtime_ns -if have_cython(): - from .platform import acl_get, acl_set - from .chunker import Chunker - from .hashindex import ChunkIndex - import msgpack -else: - import mock - msgpack = mock.Mock() +from .platform import acl_get, acl_set +from .chunker import Chunker +from .hashindex import ChunkIndex +import msgpack ITEMS_BUFFER = 1024 * 1024 diff --git a/borg/archiver.py b/borg/archiver.py index 5512bb25..ff6c4b3b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -20,16 +20,15 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ + dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .logger import create_logger, setup_logging logger = create_logger() -if have_cython(): - from .compress import Compressor, COMPR_BUFFER - from .upgrader import AtticRepositoryUpgrader - from .repository import Repository - from .cache import Cache - from .key import key_creator +from .compress import Compressor, COMPR_BUFFER +from .upgrader import AtticRepositoryUpgrader +from .repository import Repository +from .cache import Cache +from .key import key_creator from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS from .remote import RepositoryServer, RemoteRepository diff --git a/borg/cache.py b/borg/cache.py index 38ef4208..89aedd4f 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -10,12 +10,11 @@ from .key import PlaintextKey from .logger import create_logger logger = create_logger() from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \ - bigint_to_int, format_file_size, have_cython, yes + bigint_to_int, format_file_size, yes from .locking import UpgradableLock from .hashindex import ChunkIndex -if have_cython(): - import msgpack +import msgpack class Cache: diff --git a/borg/fuse.py b/borg/fuse.py index 417811fe..19dc09cc 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -7,11 +7,10 @@ import stat import tempfile import time from .archive import Archive -from .helpers import daemonize, have_cython +from .helpers import daemonize from .remote import cache_if_remote -if have_cython(): - import msgpack +import msgpack # Does this version of llfuse support ns precision? have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') diff --git a/borg/helpers.py b/borg/helpers.py index 84379e75..1937c9bf 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -22,27 +22,11 @@ from fnmatch import translate from operator import attrgetter -def have_cython(): - """allow for a way to disable Cython includes - - this is used during usage docs build, in setup.py. It is to avoid - loading the Cython libraries which are built, but sometimes not in - the search path (namely, during Tox runs). - - we simply check an environment variable (``BORG_CYTHON_DISABLE``) - which, when set (to anything) will disable includes of Cython - libraries in key places to enable usage docs to be built. - - :returns: True if Cython is available, False otherwise. - """ - return not os.environ.get('BORG_CYTHON_DISABLE') - -if have_cython(): - from . import hashindex - from . import chunker - from . import crypto - import msgpack - import msgpack.fallback +from . import hashindex +from . import chunker +from . import crypto +import msgpack +import msgpack.fallback # return codes returned by borg command diff --git a/borg/key.py b/borg/key.py index 78bfa8b7..a712cd13 100644 --- a/borg/key.py +++ b/borg/key.py @@ -7,14 +7,13 @@ import textwrap import hmac from hashlib import sha256 -from .helpers import IntegrityError, get_keys_dir, Error, have_cython +from .helpers import IntegrityError, get_keys_dir, Error from .logger import create_logger logger = create_logger() -if have_cython(): - from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks - from .compress import Compressor, COMPR_BUFFER - import msgpack +from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks +from .compress import Compressor, COMPR_BUFFER +import msgpack PREFIX = b'\0' * 8 diff --git a/borg/remote.py b/borg/remote.py index 82353fef..df435ebc 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -10,11 +10,10 @@ import traceback from . import __version__ -from .helpers import Error, IntegrityError, have_cython +from .helpers import Error, IntegrityError from .repository import Repository -if have_cython(): - import msgpack +import msgpack BUFSIZE = 10 * 1024 * 1024 diff --git a/borg/repository.py b/borg/repository.py index 048fa032..0127bca3 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -10,9 +10,8 @@ import shutil import struct from zlib import crc32 -from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify, have_cython -if have_cython(): - from .hashindex import NSIndex +from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify +from .hashindex import NSIndex from .locking import UpgradableLock from .lrucache import LRUCache diff --git a/docs/usage.rst b/docs/usage.rst index 46ecd4bf..abeb074a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -59,12 +59,6 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do For "Warning: The repository at location ... was previously located at ..." BORG_CHECK_I_KNOW_WHAT_I_AM_DOING For "Warning: 'check --repair' is an experimental feature that might result in data loss." - BORG_CYTHON_DISABLE - Disables the loading of Cython modules. This is currently - experimental and is used only to generate usage docs at build - time. It is unlikely to produce good results on a regular - run. The variable should be set to the name of the calling class, and - should be unique across all of borg. It is currently only used by ``build_usage``. Directories: BORG_KEYS_DIR diff --git a/setup.py b/setup.py index 5c6e98a2..cf699447 100644 --- a/setup.py +++ b/setup.py @@ -135,8 +135,6 @@ class build_usage(Command): def run(self): print('generating usage docs') # allows us to build docs without the C modules fully loaded during help generation - if 'BORG_CYTHON_DISABLE' not in os.environ: - os.environ['BORG_CYTHON_DISABLE'] = self.__class__.__name__ from borg.archiver import Archiver parser = Archiver().build_parser(prog='borg') choices = {} @@ -166,9 +164,6 @@ class build_usage(Command): doc.write(re.sub("^", " ", parser.format_help(), flags=re.M)) doc.write("\nDescription\n~~~~~~~~~~~\n") doc.write(epilog) - # return to regular Cython configuration, if we changed it - if os.environ.get('BORG_CYTHON_DISABLE') == self.__class__.__name__: - del os.environ['BORG_CYTHON_DISABLE'] class build_api(Command): From 8e36075fe95edcfd824ab61901a3fd279bfe21c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 13 Nov 2015 10:42:16 -0500 Subject: [PATCH 062/321] commit usage files directly into git the generation of those files was causing us way too much pain to justify automatically generating them all the time. those will have to be re-generated with `build_api` or `build_usage` as appropriate, for example when function signatures or commandline flags change. see #384 --- .gitignore | 2 - docs/api.rst | 87 +++++++++++++++++++++ docs/usage/change-passphrase.rst.inc | 29 +++++++ docs/usage/check.rst.inc | 67 ++++++++++++++++ docs/usage/create.rst.inc | 73 +++++++++++++++++ docs/usage/debug-delete-obj.rst.inc | 29 +++++++ docs/usage/debug-dump-archive-items.rst.inc | 28 +++++++ docs/usage/debug-get-obj.rst.inc | 30 +++++++ docs/usage/debug-put-obj.rst.inc | 29 +++++++ docs/usage/delete.rst.inc | 32 ++++++++ docs/usage/extract.rst.inc | 49 ++++++++++++ docs/usage/help.rst.inc | 34 ++++++++ docs/usage/info.rst.inc | 28 +++++++ docs/usage/init.rst.inc | 34 ++++++++ docs/usage/list.rst.inc | 32 ++++++++ docs/usage/mount.rst.inc | 35 +++++++++ docs/usage/prune.rst.inc | 66 ++++++++++++++++ docs/usage/rename.rst.inc | 29 +++++++ docs/usage/serve.rst.inc | 27 +++++++ docs/usage/upgrade.rst.inc | 65 +++++++++++++++ 20 files changed, 803 insertions(+), 2 deletions(-) create mode 100644 docs/api.rst create mode 100644 docs/usage/change-passphrase.rst.inc create mode 100644 docs/usage/check.rst.inc create mode 100644 docs/usage/create.rst.inc create mode 100644 docs/usage/debug-delete-obj.rst.inc create mode 100644 docs/usage/debug-dump-archive-items.rst.inc create mode 100644 docs/usage/debug-get-obj.rst.inc create mode 100644 docs/usage/debug-put-obj.rst.inc create mode 100644 docs/usage/delete.rst.inc create mode 100644 docs/usage/extract.rst.inc create mode 100644 docs/usage/help.rst.inc create mode 100644 docs/usage/info.rst.inc create mode 100644 docs/usage/init.rst.inc create mode 100644 docs/usage/list.rst.inc create mode 100644 docs/usage/mount.rst.inc create mode 100644 docs/usage/prune.rst.inc create mode 100644 docs/usage/rename.rst.inc create mode 100644 docs/usage/serve.rst.inc create mode 100644 docs/usage/upgrade.rst.inc diff --git a/.gitignore b/.gitignore index 73d508fe..2d77951b 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,6 @@ platform_linux.c *.pyc *.pyo *.so -docs/usage/*.inc -docs/api.rst .idea/ .cache/ borg/_version.py diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 00000000..e16b9f18 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,87 @@ + +API Documentation +================= + +.. automodule:: borg.key + :members: + :undoc-members: + +.. automodule:: borg.cache + :members: + :undoc-members: + +.. automodule:: borg.locking + :members: + :undoc-members: + +.. automodule:: borg.platform + :members: + :undoc-members: + +.. automodule:: borg.xattr + :members: + :undoc-members: + +.. automodule:: borg.fuse + :members: + :undoc-members: + +.. automodule:: borg.logger + :members: + :undoc-members: + +.. automodule:: borg.repository + :members: + :undoc-members: + +.. automodule:: borg.archiver + :members: + :undoc-members: + +.. automodule:: borg.archive + :members: + :undoc-members: + +.. automodule:: borg.lrucache + :members: + :undoc-members: + +.. automodule:: borg.remote + :members: + :undoc-members: + +.. automodule:: borg.upgrader + :members: + :undoc-members: + +.. automodule:: borg.helpers + :members: + :undoc-members: + +.. automodule:: borg.platform_freebsd + :members: + :undoc-members: + +.. automodule:: borg.hashindex + :members: + :undoc-members: + +.. automodule:: borg.chunker + :members: + :undoc-members: + +.. automodule:: borg.platform_darwin + :members: + :undoc-members: + +.. automodule:: borg.platform_linux + :members: + :undoc-members: + +.. automodule:: borg.compress + :members: + :undoc-members: + +.. automodule:: borg.crypto + :members: + :undoc-members: diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc new file mode 100644 index 00000000..9d0926d1 --- /dev/null +++ b/docs/usage/change-passphrase.rst.inc @@ -0,0 +1,29 @@ +.. _borg_change-passphrase: + +borg change-passphrase +---------------------- +:: + + usage: borg change-passphrase [-h] [-v] [--show-rc] [--no-files-cache] + [--umask M] [--remote-path PATH] + [REPOSITORY] + + Change repository key file passphrase + + positional arguments: + REPOSITORY + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +The key files used for repository encryption are optionally passphrase +protected. This command can be used to change this passphrase. diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc new file mode 100644 index 00000000..f2a05956 --- /dev/null +++ b/docs/usage/check.rst.inc @@ -0,0 +1,67 @@ +.. _borg_check: + +borg check +---------- +:: + + usage: borg check [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--repository-only] [--archives-only] + [--repair] [--last N] + [REPOSITORY_OR_ARCHIVE] + + Check repository consistency + + positional arguments: + REPOSITORY_OR_ARCHIVE + repository or archive to check consistency of + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + --repository-only only perform repository checks + --archives-only only perform archives checks + --repair attempt to repair any inconsistencies found + --last N only check last N archives (Default: all) + +Description +~~~~~~~~~~~ + +The check command verifies the consistency of a repository and the corresponding archives. + +First, the underlying repository data files are checked: + +- For all segments the segment magic (header) is checked +- For all objects stored in the segments, all metadata (e.g. crc and size) and + all data is read. The read data is checked by size and CRC. Bit rot and other + types of accidental damage can be detected this way. +- If we are in repair mode and a integrity error is detected for a segment, + we try to recover as many objects from the segment as possible. +- In repair mode, it makes sure that the index is consistent with the data + stored in the segments. +- If you use a remote repo server via ssh:, the repo check is executed on the + repo server without causing significant network traffic. +- The repository check can be skipped using the --archives-only option. + +Second, the consistency and correctness of the archive metadata is verified: + +- Is the repo manifest present? If not, it is rebuilt from archive metadata + chunks (this requires reading and decrypting of all metadata and data). +- Check if archive metadata chunk is present. if not, remove archive from + manifest. +- For all files (items) in the archive, for all chunks referenced by these + files, check if chunk is present (if not and we are in repair mode, replace + it with a same-size chunk of zeros). This requires reading of archive and + file metadata, but not data. +- If we are in repair mode and we checked all the archives: delete orphaned + chunks from the repo. +- if you use a remote repo server via ssh:, the archive check is executed on + the client machine (because if encryption is enabled, the checks will require + decryption and this is always done client-side, because key access will be + required). +- The archive checks can be time consuming, they can be skipped using the + --repository-only option. diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc new file mode 100644 index 00000000..c52108cb --- /dev/null +++ b/docs/usage/create.rst.inc @@ -0,0 +1,73 @@ +.. _borg_create: + +borg create +----------- +:: + + usage: borg create [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-s] [-p] [-e PATTERN] + [--exclude-from EXCLUDEFILE] [--exclude-caches] + [--exclude-if-present FILENAME] [--keep-tag-files] + [-c SECONDS] [-x] [--numeric-owner] + [--timestamp yyyy-mm-ddThh:mm:ss] + [--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE] + [-C COMPRESSION] [--read-special] [-n] + ARCHIVE PATH [PATH ...] + + Create new archive + + positional arguments: + ARCHIVE name of archive to create (must be also a valid + directory name) + PATH paths to archive + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -s, --stats print statistics for the created archive + -p, --progress toggle progress display while creating the archive, + showing Original, Compressed and Deduplicated sizes, + followed by the Number of files seen and the path + being processed, default: True + -e PATTERN, --exclude PATTERN + exclude paths matching PATTERN + --exclude-from EXCLUDEFILE + read exclude patterns from EXCLUDEFILE, one per line + --exclude-caches exclude directories that contain a CACHEDIR.TAG file + (http://www.brynosaurus.com/cachedir/spec.html) + --exclude-if-present FILENAME + exclude directories that contain the specified file + --keep-tag-files keep tag files of excluded caches/directories + -c SECONDS, --checkpoint-interval SECONDS + write checkpoint every SECONDS seconds (Default: 300) + -x, --one-file-system + stay in same file system, do not cross mount points + --numeric-owner only store numeric user and group identifiers + --timestamp yyyy-mm-ddThh:mm:ss + manually specify the archive creation date/time (UTC). + alternatively, give a reference file/directory. + --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE + specify the chunker parameters. default: 10,23,16,4095 + -C COMPRESSION, --compression COMPRESSION + select compression algorithm (and level): none == no + compression (default), lz4 == lz4, zlib == zlib + (default level 6), zlib,0 .. zlib,9 == zlib (with + level 0..9), lzma == lzma (default level 6), lzma,0 .. + lzma,9 == lzma (with level 0..9). + --read-special open and read special files as if they were regular + files + -n, --dry-run do not create a backup archive + +Description +~~~~~~~~~~~ + +This command creates a backup archive containing all files found while recursively +traversing all paths specified. The archive will consume almost no disk space for +files or parts of files that have already been stored in other archives. + +See the output of the "borg help patterns" command for more help on exclude patterns. diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc new file mode 100644 index 00000000..3cdea321 --- /dev/null +++ b/docs/usage/debug-delete-obj.rst.inc @@ -0,0 +1,29 @@ +.. _borg_debug-delete-obj: + +borg debug-delete-obj +--------------------- +:: + + usage: borg debug-delete-obj [-h] [-v] [--show-rc] [--no-files-cache] + [--umask M] [--remote-path PATH] + [REPOSITORY] IDs [IDs ...] + + delete the objects with the given IDs from the repo + + positional arguments: + REPOSITORY repository to use + IDs hex object ID(s) to delete from the repo + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command deletes objects from the repository. diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc new file mode 100644 index 00000000..cb68493d --- /dev/null +++ b/docs/usage/debug-dump-archive-items.rst.inc @@ -0,0 +1,28 @@ +.. _borg_debug-dump-archive-items: + +borg debug-dump-archive-items +----------------------------- +:: + + usage: borg debug-dump-archive-items [-h] [-v] [--show-rc] [--no-files-cache] + [--umask M] [--remote-path PATH] + ARCHIVE + + dump (decrypted, decompressed) archive items metadata (not: data) + + positional arguments: + ARCHIVE archive to dump + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc new file mode 100644 index 00000000..8343d866 --- /dev/null +++ b/docs/usage/debug-get-obj.rst.inc @@ -0,0 +1,30 @@ +.. _borg_debug-get-obj: + +borg debug-get-obj +------------------ +:: + + usage: borg debug-get-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] + [REPOSITORY] ID PATH + + get object contents from the repository and write it into file + + positional arguments: + REPOSITORY repository to use + ID hex object ID to get from the repo + PATH file to write object data into + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command gets an object from the repository. diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc new file mode 100644 index 00000000..cb14b5de --- /dev/null +++ b/docs/usage/debug-put-obj.rst.inc @@ -0,0 +1,29 @@ +.. _borg_debug-put-obj: + +borg debug-put-obj +------------------ +:: + + usage: borg debug-put-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] + [REPOSITORY] PATH [PATH ...] + + put file(s) contents into the repository + + positional arguments: + REPOSITORY repository to use + PATH file(s) to read and create object(s) from + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command puts objects into the repository. diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc new file mode 100644 index 00000000..afdbd9db --- /dev/null +++ b/docs/usage/delete.rst.inc @@ -0,0 +1,32 @@ +.. _borg_delete: + +borg delete +----------- +:: + + usage: borg delete [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-s] [-c] + [TARGET] + + Delete an existing repository or archive + + positional arguments: + TARGET archive or repository to delete + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -s, --stats print statistics for the deleted archive + -c, --cache-only delete only the local cache for the given repository + +Description +~~~~~~~~~~~ + +This command deletes an archive from the repository or the complete repository. +Disk space is reclaimed accordingly. If you delete the complete repository, the +local cache for it (if any) is also deleted. diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc new file mode 100644 index 00000000..cbacd703 --- /dev/null +++ b/docs/usage/extract.rst.inc @@ -0,0 +1,49 @@ +.. _borg_extract: + +borg extract +------------ +:: + + usage: borg extract [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-n] [-e PATTERN] + [--exclude-from EXCLUDEFILE] [--numeric-owner] + [--strip-components NUMBER] [--stdout] [--sparse] + ARCHIVE [PATH [PATH ...]] + + Extract archive contents + + positional arguments: + ARCHIVE archive to extract + PATH paths to extract + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -n, --dry-run do not actually change any files + -e PATTERN, --exclude PATTERN + exclude paths matching PATTERN + --exclude-from EXCLUDEFILE + read exclude patterns from EXCLUDEFILE, one per line + --numeric-owner only obey numeric user and group identifiers + --strip-components NUMBER + Remove the specified number of leading path elements. + Pathnames with fewer elements will be silently + skipped. + --stdout write all extracted data to stdout + --sparse create holes in output sparse file from all-zero + chunks + +Description +~~~~~~~~~~~ + +This command extracts the contents of an archive. By default the entire +archive is extracted but a subset of files and directories can be selected +by passing a list of ``PATHs`` as arguments. The file selection can further +be restricted by using the ``--exclude`` option. + +See the output of the "borg help patterns" command for more help on exclude patterns. diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc new file mode 100644 index 00000000..0379cd49 --- /dev/null +++ b/docs/usage/help.rst.inc @@ -0,0 +1,34 @@ +.. _borg_patterns: + +borg help patterns +~~~~~~~~~~~~~~~~~~ +:: + + + Exclude patterns use a variant of shell pattern syntax, with '*' matching any + number of characters, '?' matching any single character, '[...]' matching any + single character specified, including ranges, and '[!...]' matching any + character not specified. For the purpose of these patterns, the path + separator ('\' for Windows and '/' on other systems) is not treated + specially. For a path to match a pattern, it must completely match from + start to end, or must match from the start to just before a path separator. + Except for the root path, paths will never end in the path separator when + matching is attempted. Thus, if a given pattern ends in a path separator, a + '*' is appended before matching is attempted. Patterns with wildcards should + be quoted to protect them from shell expansion. + + Examples: + + # Exclude '/home/user/file.o' but not '/home/user/file.odt': + $ borg create -e '*.o' backup / + + # Exclude '/home/user/junk' and '/home/user/subdir/junk' but + # not '/home/user/importantjunk' or '/etc/junk': + $ borg create -e '/home/*/junk' backup / + + # Exclude the contents of '/home/user/cache' but not the directory itself: + $ borg create -e /home/user/cache/ backup / + + # The file '/home/user/cache/important' is *not* backed up: + $ borg create -e /home/user/cache/ backup / /home/user/cache/important + \ No newline at end of file diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc new file mode 100644 index 00000000..2ddb651e --- /dev/null +++ b/docs/usage/info.rst.inc @@ -0,0 +1,28 @@ +.. _borg_info: + +borg info +--------- +:: + + usage: borg info [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] + ARCHIVE + + Show archive details such as disk space used + + positional arguments: + ARCHIVE archive to display information about + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command displays some detailed information about the specified archive. diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc new file mode 100644 index 00000000..4e0f47b0 --- /dev/null +++ b/docs/usage/init.rst.inc @@ -0,0 +1,34 @@ +.. _borg_init: + +borg init +--------- +:: + + usage: borg init [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-e {none,keyfile,repokey,passphrase}] + [REPOSITORY] + + Initialize an empty repository + + positional arguments: + REPOSITORY repository to create + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -e {none,keyfile,repokey,passphrase}, --encryption {none,keyfile,repokey,passphrase} + select encryption key mode + +Description +~~~~~~~~~~~ + +This command initializes an empty repository. A repository is a filesystem +directory containing the deduplicated data from zero or more archives. +Encryption can be enabled at repository init time. +Please note that the 'passphrase' encryption mode is DEPRECATED (instead of it, +consider using 'repokey'). diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc new file mode 100644 index 00000000..882fff25 --- /dev/null +++ b/docs/usage/list.rst.inc @@ -0,0 +1,32 @@ +.. _borg_list: + +borg list +--------- +:: + + usage: borg list [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--short] [-p PREFIX] + [REPOSITORY_OR_ARCHIVE] + + List archive or repository contents + + positional arguments: + REPOSITORY_OR_ARCHIVE + repository/archive to list contents of + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + --short only print file/directory names, nothing else + -p PREFIX, --prefix PREFIX + only consider archive names starting with this prefix + +Description +~~~~~~~~~~~ + +This command lists the contents of a repository or an archive. diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc new file mode 100644 index 00000000..0407a095 --- /dev/null +++ b/docs/usage/mount.rst.inc @@ -0,0 +1,35 @@ +.. _borg_mount: + +borg mount +---------- +:: + + usage: borg mount [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-f] [-o OPTIONS] + REPOSITORY_OR_ARCHIVE MOUNTPOINT + + Mount archive or an entire repository as a FUSE fileystem + + positional arguments: + REPOSITORY_OR_ARCHIVE + repository/archive to mount + MOUNTPOINT where to mount filesystem + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -f, --foreground stay in foreground, do not daemonize + -o OPTIONS Extra mount options + +Description +~~~~~~~~~~~ + +This command mounts an archive as a FUSE filesystem. This can be useful for +browsing an archive or restoring individual files. Unless the ``--foreground`` +option is given the command will run in the background until the filesystem +is ``umounted``. diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc new file mode 100644 index 00000000..c2133736 --- /dev/null +++ b/docs/usage/prune.rst.inc @@ -0,0 +1,66 @@ +.. _borg_prune: + +borg prune +---------- +:: + + usage: borg prune [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-n] [-s] [--keep-within WITHIN] + [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY] + [-p PREFIX] + [REPOSITORY] + + Prune repository archives according to specified rules + + positional arguments: + REPOSITORY repository to prune + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -n, --dry-run do not change repository + -s, --stats print statistics for the deleted archive + --keep-within WITHIN keep all archives within this time interval + -H HOURLY, --keep-hourly HOURLY + number of hourly archives to keep + -d DAILY, --keep-daily DAILY + number of daily archives to keep + -w WEEKLY, --keep-weekly WEEKLY + number of weekly archives to keep + -m MONTHLY, --keep-monthly MONTHLY + number of monthly archives to keep + -y YEARLY, --keep-yearly YEARLY + number of yearly archives to keep + -p PREFIX, --prefix PREFIX + only consider archive names starting with this prefix + +Description +~~~~~~~~~~~ + +The prune command prunes a repository by deleting archives not matching +any of the specified retention options. This command is normally used by +automated backup scripts wanting to keep a certain number of historic backups. + +As an example, "-d 7" means to keep the latest backup on each day for 7 days. +Days without backups do not count towards the total. +The rules are applied from hourly to yearly, and backups selected by previous +rules do not count towards those of later rules. The time that each backup +completes is used for pruning purposes. Dates and times are interpreted in +the local timezone, and weeks go from Monday to Sunday. Specifying a +negative number of archives to keep means that there is no limit. + +The "--keep-within" option takes an argument of the form "", +where char is "H", "d", "w", "m", "y". For example, "--keep-within 2d" means +to keep all archives that were created within the past 48 hours. +"1m" is taken to mean "31d". The archives kept with this option do not +count towards the totals specified by any other options. + +If a prefix is set with -p, then only archives that start with the prefix are +considered for deletion and only those archives count towards the totals +specified by the rules. +Otherwise, *all* archives in the repository are candidates for deletion! diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc new file mode 100644 index 00000000..23cb338e --- /dev/null +++ b/docs/usage/rename.rst.inc @@ -0,0 +1,29 @@ +.. _borg_rename: + +borg rename +----------- +:: + + usage: borg rename [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] + ARCHIVE NEWNAME + + Rename an existing archive + + positional arguments: + ARCHIVE archive to rename + NEWNAME the new archive name to use + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command renames an archive in the repository. diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc new file mode 100644 index 00000000..8c405226 --- /dev/null +++ b/docs/usage/serve.rst.inc @@ -0,0 +1,27 @@ +.. _borg_serve: + +borg serve +---------- +:: + + usage: borg serve [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--restrict-to-path PATH] + + Start in server mode. This command is usually not used manually. + + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + --restrict-to-path PATH + restrict repository access to PATH + +Description +~~~~~~~~~~~ + +This command starts a repository server process. This command is usually not used manually. diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc new file mode 100644 index 00000000..7b9660b5 --- /dev/null +++ b/docs/usage/upgrade.rst.inc @@ -0,0 +1,65 @@ +.. _borg_upgrade: + +borg upgrade +------------ +:: + + usage: borg upgrade [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-n] [-i] + [REPOSITORY] + + upgrade a repository from a previous version + + positional arguments: + REPOSITORY path to the repository to be upgraded + + optional arguments: + -h, --help show this help message and exit + -v, --verbose verbose output + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 63) + --remote-path PATH set remote path to executable (default: "borg") + -n, --dry-run do not change repository + -i, --inplace rewrite repository in place, with no chance of going + back to older versions of the repository. + +Description +~~~~~~~~~~~ + +upgrade an existing Borg repository. this currently +only support converting an Attic repository, but may +eventually be extended to cover major Borg upgrades as well. + +it will change the magic strings in the repository's segments +to match the new Borg magic strings. the keyfiles found in +$ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and +copied to $BORG_KEYS_DIR or ~/.borg/keys. + +the cache files are converted, from $ATTIC_CACHE_DIR or +~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the +cache layout between Borg and Attic changed, so it is possible +the first backup after the conversion takes longer than expected +due to the cache resync. + +upgrade should be able to resume if interrupted, although it +will still iterate over all segments. if you want to start +from scratch, use `borg delete` over the copied repository to +make sure the cache files are also removed: + + borg delete borg + +unless ``--inplace`` is specified, the upgrade process first +creates a backup copy of the repository, in +REPOSITORY.upgrade-DATETIME, using hardlinks. this takes +longer than in place upgrades, but is much safer and gives +progress information (as opposed to ``cp -al``). once you are +satisfied with the conversion, you can safely destroy the +backup copy. + +WARNING: running the upgrade in place will make the current +copy unusable with older version, with no way of going back +to previous versions. this can PERMANENTLY DAMAGE YOUR +REPOSITORY! Attic CAN NOT READ BORG REPOSITORIES, as the +magic strings have changed. you have been warned. \ No newline at end of file From 811c18dcd4b92aed246a46c502b17864e939cb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 13 Nov 2015 10:46:13 -0500 Subject: [PATCH 063/321] explain how to regenerate usage and API files and when --- docs/development.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/development.rst b/docs/development.rst index a8d664f6..23f28385 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -61,6 +61,23 @@ Important notes: - When using -- to give options to py.test, you MUST also give borg.testsuite[.module]. +Regenerate usage files +---------------------- + +Usage and API documentation is currently committed directly to git, +although those files are generated automatically from the source +tree. + +When a new module is added, the ``docs/api.rst`` file needs to be +regenerated:: + + ./setup.py build_api + +When a command is added, a commandline flag changed, added or removed, +the usage docs need to be rebuilt as well:: + + ./setup.py build_usage + Building the docs with Sphinx ----------------------------- From fd5ccadcac920339922316e0c9ea3756e2863867 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Nov 2015 23:48:44 +0100 Subject: [PATCH 064/321] update CHANGES --- docs/changes.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index fea8bfd2..9d215e63 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,25 @@ Changelog ========= +Version 0.28.2 +-------------- + +New features: + +- borg create --exclude-if-present TAGFILE - exclude directories that have the + given file from the backup. You can additionally give --keep-tag-files to + preserve just the directory roots and the tag-files (but not backup other + directory contents), #395, attic #128, attic #142 + +Other changes: + +- do not create docs sources at build time (just have them in the repo), + completely remove have_cython() hack, do not use the "mock" library at build + time, #384 +- docs: explain how to regenerate usage and API files (build_api or + build_usage) and when to commit usage files directly into git, #384 + + Version 0.28.1 -------------- From 5942b797403c7504e5b60e4dddf0c708d48716ed Mon Sep 17 00:00:00 2001 From: jungle-boogie Date: Sat, 14 Nov 2015 16:13:25 -0800 Subject: [PATCH 065/321] correct install link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 36408513..62d3de3f 100644 --- a/README.rst +++ b/README.rst @@ -19,7 +19,7 @@ fully trusted targets. See the `installation manual`_ or, if you have already downloaded Borg, ``docs/installation.rst`` to get started with Borg. -.. _installation manual: https://borgbackup.readthedocs.org/installation.html +.. _installation manual: https://borgbackup.readthedocs.org/en/stable/installation.html Main features ------------- From 9f8700e3831d8aa445ff86fa98f1b5ea659fbb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 14 Nov 2015 21:03:38 -0500 Subject: [PATCH 066/321] restructure install page headings do not repeat "installation" all the time, and regroup git and pip under "from source" also link to the sections in the summary --- docs/installation.rst | 56 ++++++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index efe8dadb..625b1cbf 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -6,18 +6,23 @@ Installation There are different ways to install |project_name|: -- **distribution package** - easy and fast if a package is available for your - Linux/BSD distribution. -- **PyInstaller binary** - easy and fast, we provide a ready-to-use binary file +- :ref:`distribution-package` - easy and fast if a package is + available from your operating system. +- :ref:`pyinstaller-binary` - easy and fast, we provide a ready-to-use binary file that comes bundled with all dependencies. -- **pip** - installing a source package with pip needs more installation steps - and requires all dependencies with development headers and a compiler. -- **git** - for developers and power users who want to have the latest code or - use revision control (each release is tagged). +- :ref:`source-install`, either: + - :ref:`pip-installation` - installing a source package with pip needs + more installation steps and requires all dependencies with + development headers and a compiler. + - :ref:`git-installation` - for developers and power users who want to + have the latest code or use revision control (each release is + tagged). -Installation (Distribution Package) ------------------------------------ +.. _distribution-package: + +Distribution Package +-------------------- Some Linux and BSD distributions might offer a ready-to-use ``borgbackup`` package which can be installed with the package manager. As |project_name| is @@ -32,9 +37,10 @@ and compare that to our latest release and review the :doc:`changes`. .. _AUR: https://aur.archlinux.org/packages/borgbackup/ +.. _pyinstaller-binary: -Installation (PyInstaller Binary) ---------------------------------- +PyInstaller Binary +------------------ The |project_name| binary is available on the releases_ page for the following platforms: @@ -50,10 +56,15 @@ version. .. _releases: https://github.com/borgbackup/borg/releases -Installing the Dependencies ---------------------------- +.. _source-install: -To install |project_name| from a source package, you have to install the +From source +----------- + +Dependencies +~~~~~~~~~~~~ + +To install |project_name| from a source package (including pip), you have to install the following dependencies first: * `Python 3`_ >= 3.2.2. Even though Python 3 is not the default Python version on @@ -75,7 +86,7 @@ After you have installed the dependencies, you can proceed with steps outlined under :ref:`pip-installation`. Debian / Ubuntu -~~~~~~~~~~~~~~~ ++++++++++++++++ Install the dependencies with development headers:: @@ -91,7 +102,7 @@ Ubuntu this means your user is not in the ``fuse`` group. Add yourself to that group, log out and log in again. Fedora / Korora -~~~~~~~~~~~~~~~ ++++++++++++++++ Install the dependencies with development headers:: @@ -103,7 +114,7 @@ Install the dependencies with development headers:: Mac OS X -~~~~~~~~ +++++++++ Assuming you have installed homebrew_, the following steps will install all the dependencies:: @@ -117,7 +128,7 @@ FUSE for OS X, which is available as a pre-release_. .. _pre-release: https://github.com/osxfuse/osxfuse/releases Cygwin -~~~~~~ +++++++ .. note:: Running under Cygwin is experimental and has only been tested with Cygwin @@ -144,8 +155,8 @@ In case the creation of the virtual environment fails, try deleting this file:: .. _pip-installation: -Installation (pip) ------------------- +From pip +~~~~~~~~ Virtualenv_ can be used to build and install |project_name| without affecting the system Python or requiring root access. Using a virtual environment is @@ -172,9 +183,10 @@ activating your virtual environment:: pip install -U borgbackup +.. _git-installation: -Installation (git) ------------------- +From git +~~~~~~~~ This uses latest, unreleased development code from git. While we try not to break master, there are no guarantees on anything. :: From a8d2c18fee6659a4992d444ecc838d9d8c44fc73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 14 Nov 2015 21:07:16 -0500 Subject: [PATCH 067/321] split packaging meta-info in a separate paragraph and link to github --- docs/installation.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 625b1cbf..c8db6fae 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -27,16 +27,19 @@ Distribution Package Some Linux and BSD distributions might offer a ready-to-use ``borgbackup`` package which can be installed with the package manager. As |project_name| is still a young project, such a package might be not available for your system -yet. Please ask package maintainers to build a package or, if you can package / -submit it yourself, please help us with that! +yet. * On **Arch Linux**, there is a package available in the AUR_. +.. _AUR: https://aur.archlinux.org/packages/borgbackup/ + +Please ask package maintainers to build a package or, if you can package / +submit it yourself, please help us with that! See :issue:`105` on +github to followup on packaging efforts. + If a package is available, it might be interesting to check its version and compare that to our latest release and review the :doc:`changes`. -.. _AUR: https://aur.archlinux.org/packages/borgbackup/ - .. _pyinstaller-binary: PyInstaller Binary From aa4c76872c3e58ecb3d596069cc381352972a5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 14 Nov 2015 21:08:28 -0500 Subject: [PATCH 068/321] move pyinstaller mention out of heading pyinstaller probably means nothing for most people, while standalone binary is more meaningful --- docs/installation.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index c8db6fae..2506d4ec 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -42,11 +42,11 @@ and compare that to our latest release and review the :doc:`changes`. .. _pyinstaller-binary: -PyInstaller Binary ------------------- +Standalone binary +----------------- -The |project_name| binary is available on the releases_ page for the following -platforms: +|project_name| binaries (generated with `pyinstaller`_) are available +on the releases_ page for the following platforms: * **Linux**: glibc >= 2.13 (ok for most supported Linux releases) * **Mac OS X**: 10.10 (unknown whether it works for older releases) @@ -57,6 +57,7 @@ them into a directory in your ``PATH`` and then you can run ``borg``. If a new version is released, you will have to manually download it and replace the old version. +.. _pyinstaller: http://www.pyinstaller.org .. _releases: https://github.com/borgbackup/borg/releases .. _source-install: From d6b8de943b35349633d6e01876c1fe3dc3405234 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Nov 2015 14:31:25 +0100 Subject: [PATCH 069/321] add docs for item flags / status output, fixes #402 --- borg/archiver.py | 7 ------- docs/usage.rst | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index ff6c4b3b..9d65bec7 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -245,13 +245,6 @@ class Archiver: self.print_warning('Unknown file type: %s', path) return # Status output - # A lowercase character means a file type other than a regular file, - # borg usually just stores them. E.g. (d)irectory. - # Hardlinks to already seen content are indicated by (h). - # A uppercase character means a regular file that was (A)dded, - # (M)odified or was (U)nchanged. - # Note: A/M/U is relative to the "files" cache, not to the repo. - # This would be an issue if the files cache is not used. if status is None: if not dry_run: status = '?' # need to add a status code somewhere diff --git a/docs/usage.rst b/docs/usage.rst index abeb074a..16098568 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -411,6 +411,39 @@ Additional Notes Here are misc. notes about topics that are maybe not covered in enough detail in the usage section. +Item flags +~~~~~~~~~~ + +`borg create -v` outputs a verbose list of all files, directories and other +file system items it considered. For each item, it prefixes a single-letter +flag that indicates type and/or status of the item. + +A uppercase character represents the status of a regular file relative to the +"files" cache (not relative to the repo - this is an issue if the files cache +is not used). Metadata is stored in any case and for 'A' and 'M' also new data +chunks are stored. For 'U' all data chunks refer to already existing chunks. + +- 'A' = regular file, added +- 'M' = regular file, modified +- 'U' = regular file, unchanged + +A lowercase character means a file type other than a regular file, +borg usually just stores their metadata: + +- 'd' = directory +- 'b' = block device +- 'c' = char device +- 'h' = regular file, hardlink (to already seen inodes) +- 's' = symlink +- 'f' = fifo + +Other flags used include: + +- 'i' = backup data was read from standard input (stdin) +- '-' = dry run, item was *not* backed up +- '?' = missing status code (if you see this, please file a bug report!) + + --chunker-params ~~~~~~~~~~~~~~~~ The chunker params influence how input files are cut into pieces (chunks) From 234a88bec663d628f7b03847e710eb6bd5bc1d8b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Nov 2015 15:52:02 +0100 Subject: [PATCH 070/321] avoid hidden import, make it easy for pyinstaller this fixes #218 in an easier way so one doesn't have to type --hidden-import=logging.config all the time when using pyinstaller. --- Vagrantfile | 2 +- borg/logger.py | 4 ++++ docs/development.rst | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 02b43c91..dbb73f37 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -237,7 +237,7 @@ def build_binary_with_pyinstaller(boxname) cd /vagrant/borg . borg-env/bin/activate cd borg - pyinstaller -F -n borg.exe --distpath=/vagrant/borg --clean --hidden-import=logging.config borg/__main__.py + pyinstaller -F -n borg.exe --distpath=/vagrant/borg --clean borg/__main__.py EOF end diff --git a/borg/logger.py b/borg/logger.py index 6c5d2ff8..2d4b49bc 100644 --- a/borg/logger.py +++ b/borg/logger.py @@ -33,6 +33,10 @@ The way to use this is as follows: import inspect import logging +# make it easy for PyInstaller (it does not discover the dependency on this +# module automatically, because it is lazy-loaded by logging, see #218): +import logging.config + def setup_logging(stream=None): """setup logging module according to the arguments provided diff --git a/docs/development.rst b/docs/development.rst index 23f28385..f51ea414 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -129,7 +129,7 @@ When using the Vagrant VMs, pyinstaller will already be installed. With virtual env activated:: pip install pyinstaller>=3.0 # or git checkout master - pyinstaller -F -n borg-PLATFORM --hidden-import=logging.config borg/__main__.py + pyinstaller -F -n borg-PLATFORM borg/__main__.py for file in dist/borg-*; do gpg --armor --detach-sign $file; done If you encounter issues, see also our `Vagrantfile` for details. From 0070ef0c4a5fe80213962f5b6d8544b16d17ee1f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Nov 2015 20:23:33 +0100 Subject: [PATCH 071/321] minor install docs fixes --- docs/installation.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 2506d4ec..33816144 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -7,7 +7,7 @@ Installation There are different ways to install |project_name|: - :ref:`distribution-package` - easy and fast if a package is - available from your operating system. + available from your distribution / for your operating system. - :ref:`pyinstaller-binary` - easy and fast, we provide a ready-to-use binary file that comes bundled with all dependencies. - :ref:`source-install`, either: @@ -42,7 +42,7 @@ and compare that to our latest release and review the :doc:`changes`. .. _pyinstaller-binary: -Standalone binary +Standalone Binary ----------------- |project_name| binaries (generated with `pyinstaller`_) are available @@ -62,7 +62,7 @@ version. .. _source-install: -From source +From Source ----------- Dependencies @@ -159,8 +159,8 @@ In case the creation of the virtual environment fails, try deleting this file:: .. _pip-installation: -From pip -~~~~~~~~ +From PyPi / using pip +~~~~~~~~~~~~~~~~~~~~~ Virtualenv_ can be used to build and install |project_name| without affecting the system Python or requiring root access. Using a virtual environment is @@ -189,8 +189,8 @@ activating your virtual environment:: .. _git-installation: -From git -~~~~~~~~ +From Github / using git +~~~~~~~~~~~~~~~~~~~~~~~ This uses latest, unreleased development code from git. While we try not to break master, there are no guarantees on anything. :: From 3a72fbe418e075ec7af8695a50dee50603d09b0d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Nov 2015 20:30:58 +0100 Subject: [PATCH 072/321] update CHANGES --- docs/changes.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 9d215e63..f60edd59 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -16,8 +16,13 @@ Other changes: - do not create docs sources at build time (just have them in the repo), completely remove have_cython() hack, do not use the "mock" library at build time, #384 -- docs: explain how to regenerate usage and API files (build_api or - build_usage) and when to commit usage files directly into git, #384 +- avoid hidden import, make it easier for PyInstaller, easier fix for #218 +- docs: + + - add description of item flags / status output, fixes #402 + - explain how to regenerate usage and API files (build_api or + build_usage) and when to commit usage files directly into git, #384 + - minor install docs improvements Version 0.28.1 From 2c3f3f1b07253371795a14d7674d1693d4355f9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 15 Nov 2015 22:20:07 -0500 Subject: [PATCH 073/321] install docs: don't use "from" twice it always abstracts "pypi" and "github" away (technical words that are not necessary). pip remains. --- docs/installation.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 33816144..7a3f4050 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -159,8 +159,8 @@ In case the creation of the virtual environment fails, try deleting this file:: .. _pip-installation: -From PyPi / using pip -~~~~~~~~~~~~~~~~~~~~~ +Using pip +~~~~~~~~~ Virtualenv_ can be used to build and install |project_name| without affecting the system Python or requiring root access. Using a virtual environment is @@ -189,8 +189,8 @@ activating your virtual environment:: .. _git-installation: -From Github / using git -~~~~~~~~~~~~~~~~~~~~~~~ +Using git +~~~~~~~~~ This uses latest, unreleased development code from git. While we try not to break master, there are no guarantees on anything. :: From d78d6d60b1de5db53b8972a6c9b70cb3dc6dac8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 15 Nov 2015 22:40:00 -0500 Subject: [PATCH 074/321] turn distro list into a table, adding Debian, Ubuntu, OSX --- docs/installation.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 7a3f4050..f2959c03 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -29,9 +29,19 @@ package which can be installed with the package manager. As |project_name| is still a young project, such a package might be not available for your system yet. -* On **Arch Linux**, there is a package available in the AUR_. +================ ===================== ======= +Operating system Source Command +================ ===================== ======= +Arch Linux AUR_ +Debian `unstable/sid`_ ``apt install borgbackup`` +Ubuntu `Xenial Xerus 15.04`_ ``apt install borgbackup`` +OS X `Brew cask`_ ``brew cask install borgbackup`` +================ ===================== ======= .. _AUR: https://aur.archlinux.org/packages/borgbackup/ +.. _unstable/sid: https://packages.debian.org/sid/borgbackup +.. _Xenial Xerus 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup +.. _Brew cask: http://caskroom.io/ Please ask package maintainers to build a package or, if you can package / submit it yourself, please help us with that! See :issue:`105` on From 3bbd3319a441551e07b12ececba93cd6398a8e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 15 Nov 2015 22:42:49 -0500 Subject: [PATCH 075/321] use 'distribution' more consistently --- docs/installation.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index f2959c03..d7af1869 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -7,7 +7,7 @@ Installation There are different ways to install |project_name|: - :ref:`distribution-package` - easy and fast if a package is - available from your distribution / for your operating system. + available from your distribution. - :ref:`pyinstaller-binary` - easy and fast, we provide a ready-to-use binary file that comes bundled with all dependencies. - :ref:`source-install`, either: @@ -24,19 +24,19 @@ There are different ways to install |project_name|: Distribution Package -------------------- -Some Linux and BSD distributions might offer a ready-to-use ``borgbackup`` +Some distributions might offer a ready-to-use ``borgbackup`` package which can be installed with the package manager. As |project_name| is still a young project, such a package might be not available for your system yet. -================ ===================== ======= -Operating system Source Command -================ ===================== ======= -Arch Linux AUR_ -Debian `unstable/sid`_ ``apt install borgbackup`` -Ubuntu `Xenial Xerus 15.04`_ ``apt install borgbackup`` -OS X `Brew cask`_ ``brew cask install borgbackup`` -================ ===================== ======= +============ ===================== ======= +Distribution Source Command +============ ===================== ======= +Arch Linux AUR_ +Debian `unstable/sid`_ ``apt install borgbackup`` +Ubuntu `Xenial Xerus 15.04`_ ``apt install borgbackup`` +OS X `Brew cask`_ ``brew cask install borgbackup`` +============ ===================== ======= .. _AUR: https://aur.archlinux.org/packages/borgbackup/ .. _unstable/sid: https://packages.debian.org/sid/borgbackup From 917236231f452901cf00a278ff763a7cbe5a7977 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 16 Nov 2015 09:07:59 -0500 Subject: [PATCH 076/321] correct ubuntu release number --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index d7af1869..c6eb149b 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -34,7 +34,7 @@ Distribution Source Command ============ ===================== ======= Arch Linux AUR_ Debian `unstable/sid`_ ``apt install borgbackup`` -Ubuntu `Xenial Xerus 15.04`_ ``apt install borgbackup`` +Ubuntu `Xenial Xerus 16.04`_ ``apt install borgbackup`` OS X `Brew cask`_ ``brew cask install borgbackup`` ============ ===================== ======= From 38472900af82ab18f825fcf8c240302b63116844 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 16 Nov 2015 15:37:29 +0100 Subject: [PATCH 077/321] add 'E' file status to docs --- docs/usage.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/usage.rst b/docs/usage.rst index 16098568..a585ee2f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -426,6 +426,7 @@ chunks are stored. For 'U' all data chunks refer to already existing chunks. - 'A' = regular file, added - 'M' = regular file, modified - 'U' = regular file, unchanged +- 'E' = regular file, an error happened while accessing/reading *this* file A lowercase character means a file type other than a regular file, borg usually just stores their metadata: From 5be060d1f1e904e157b9fe6d221046652a0291cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 16 Nov 2015 11:27:27 -0500 Subject: [PATCH 078/321] add a --no-progress flag to forcibly disable progress info --progress isn't a "toggle" anymore, in that it will never disable progress information: always enable it. example: $ borg create ~/test/borg2::test$(date +%s) . ; echo ^shows progress reading files cache processing files ^shows progress $ borg create ~/test/borg2::test$(date +%s) . < /dev/null; echo ^no progress reading files cache processing files ^no progress $ borg create --progress ~/test/borg2::test$(date +%s) . < /dev/null; echo ^progress forced reading files cache processing files ^progress forced $ borg create --no-progress ~/test/borg2::test$(date +%s) . ; echo ^no progress reading files cache processing files ^no progress we introduce a ToggleAction that can be used for other options, but right now is just slapped in there near the code, which isn't that elegant. inspired by: http://stackoverflow.com/questions/11507756/python-argparse-toggle-flags note that this is supported out of the box by click: http://click.pocoo.org/5/options/#boolean-flags fixes #398 --- borg/archiver.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9d65bec7..c1de0537 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -761,6 +761,19 @@ class Archiver: See the output of the "borg help patterns" command for more help on exclude patterns. """) + class ToggleAction(argparse.Action): + """argparse action to handle "toggle" flags easily + + toggle flags are in the form of ``--foo``, ``--no-foo``. + + the ``--no-foo`` argument still needs to be passed to the + ``add_argument()`` call, but it simplifies the ``--no`` + detection. + """ + def __call__(self, parser, ns, values, option): + """set the given flag to true unless ``--no`` is passed""" + setattr(ns, self.dest, not '--no' in option) + subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog, @@ -769,8 +782,9 @@ class Archiver: subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the created archive') - subparser.add_argument('-p', '--progress', dest='progress', const=not sys.stderr.isatty(), - action='store_const', default=sys.stdin.isatty(), + subparser.add_argument('-p', '--progress', '--no-progress', + dest='progress', default=sys.stdin.isatty(), + action=ToggleAction, nargs=0, help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: %(default)s""") From 9b1ca5c1eba9b353fa63dca582fdc5a7b4785dc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 16 Nov 2015 15:26:50 -0500 Subject: [PATCH 079/321] force --no to be at the start of option --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index c1de0537..93de2931 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -772,7 +772,7 @@ class Archiver: """ def __call__(self, parser, ns, values, option): """set the given flag to true unless ``--no`` is passed""" - setattr(ns, self.dest, not '--no' in option) + setattr(ns, self.dest, option.startswith('--no-')) subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__, From 559c8908c1b6037c338c084511c789e97fb3da40 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 16 Nov 2015 16:03:55 +0100 Subject: [PATCH 080/321] add FAQ entry about unexpected 'A' status for unchanged file(s), fixes #403 --- docs/faq.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/faq.rst b/docs/faq.rst index d98d2625..5487532b 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -165,6 +165,32 @@ Yes, if you want to detect accidental data damage (like bit rot), use the If you want to be able to detect malicious tampering also, use a encrypted repo. It will then be able to check using CRCs and HMACs. +I am seeing 'A' (added) status for a unchanged file!? +----------------------------------------------------- + +The files cache (which is used to determine whether |project_name| already +"knows" / has backed up a file and if so, to skip the file from chunking) +does intentionally *not* contain files that: + +- have >= 10 as "entry age" (|project_name| has not seen this file for a while) +- have a modification time (mtime) same as the newest mtime in the created + archive + +So, if you see an 'A' status for unchanged file(s), they are likely the files +with the most recent mtime in that archive. + +This is expected: it is to avoid data loss with files that are backed up from +a snapshot and that are immediately changed after the snapshot (but within +mtime granularity time, so the mtime would not change). Without the code that +removes these files from the files cache, the change that happened right after +the snapshot would not be contained in the next backup as |project_name| would +think the file is unchanged. + +This does not affect deduplication, the file will be chunked, but as the chunks +will often be the same and already stored in the repo (except in the above +mentioned rare condition), it will just re-use them as usual and not store new +data chunks. + Why was Borg forked from Attic? ------------------------------- From 2e64c29e014391394898bbac05fc0c19bd1453b0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 16 Nov 2015 23:51:21 +0100 Subject: [PATCH 081/321] use ISO-8601 date and time format, fixes #375 --- borg/archiver.py | 2 +- borg/helpers.py | 9 +++------ docs/conf.py | 4 ++-- docs/usage.rst | 8 ++++++++ 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9d65bec7..f7090b0c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -431,7 +431,7 @@ class Archiver: print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) print('Hostname:', archive.metadata[b'hostname']) print('Username:', archive.metadata[b'username']) - print('Time: %s' % to_localtime(archive.ts).strftime('%c')) + print('Time: %s' % format_time(to_localtime(archive.ts))) print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) print('Number of files: %d' % stats.nfiles) print() diff --git a/borg/helpers.py b/borg/helpers.py index 1937c9bf..e20aca6e 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -449,12 +449,9 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): def format_time(t): - """Format datetime suitable for fixed length list output + """use ISO-8601 date and time format """ - if abs((datetime.now() - t).days) < 365: - return t.strftime('%b %d %H:%M') - else: - return t.strftime('%b %d %Y') + return t.strftime('%Y-%m-%d %H:%M:%S') def format_timedelta(td): @@ -510,7 +507,7 @@ def sizeof_fmt_decimal(num, suffix='B', sep='', precision=2): def format_archive(archive): - return '%-36s %s' % (archive.name, to_localtime(archive.ts).strftime('%c')) + return '%-36s %s' % (archive.name, format_time(to_localtime(archive.ts))) def memoize(function): diff --git a/docs/conf.py b/docs/conf.py index db9cd2de..237f21ad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -63,7 +63,7 @@ release = version # non-false value, then it is used: #today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +today_fmt = '%Y-%m-%d' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -131,7 +131,7 @@ html_favicon = '_static/favicon.ico' # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%Y-%m-%d' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/docs/usage.rst b/docs/usage.rst index a585ee2f..7a06ec8f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -140,6 +140,14 @@ indicated using the `IEC binary prefixes using powers of two (so ``KiB`` means 1024 bytes). +Date and Time +~~~~~~~~~~~~~ + +We format date and time conforming to ISO-8601, that is: YYYY-MM-DD and HH:MM:SS + +For more information, see: https://xkcd.com/1179/ + + .. include:: usage/init.rst.inc Examples From dcab7dd8a7413a39c5887a75b47148920f93c0fa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 18 Nov 2015 15:40:23 +0100 Subject: [PATCH 082/321] new mailing list borgbackup@python.org also: remove mailing list and irc channel address from development docs, it is enough to have this information on the main page and on the support page. --- README.rst | 3 ++- docs/development.rst | 4 ++-- docs/support.rst | 15 ++++++++------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 62d3de3f..35acf491 100644 --- a/README.rst +++ b/README.rst @@ -121,7 +121,8 @@ Links * `GitHub `_ * `Issue Tracker `_ * `Bounties & Fundraisers `_ - * `Mailing List `_ + * `New Mailing List `_ + * `(Old Mailing List's Archives `_) * `License `_ Related Projects diff --git a/docs/development.rst b/docs/development.rst index f51ea414..def08b2d 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -164,9 +164,9 @@ Checklist: - close release milestone on Github - announce on: - - `mailing list `_ + - Mailing list - Twitter (follow @ThomasJWaldmann for these tweets) - - `IRC channel `_ (change ``/topic``) + - IRC channel (change ``/topic``) - create a Github release, include: * standalone binaries (see above for how to create them) diff --git a/docs/support.rst b/docs/support.rst index e5986267..503fb81e 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -26,15 +26,16 @@ Stay connected. Mailing list ------------ -There is a mailing list for Borg on librelist_ that you can use for feature -requests and general discussions about Borg. A mailing list archive is -available `here `_. +New Mailing List: -To subscribe to the list, send an email to borgbackup@librelist.com and reply -to the confirmation mail. +`borgbackup@python.org `_ -To unsubscribe, send an email to borgbackup-unsubscribe@librelist.com and reply -to the confirmation mail. +Just read that page to find out about the mailing list, its topic, how to subscribe, +how to unsubscribe and where you can find the archives of the list. + +We used a different mailing list before, you can still read its archives there: + +`Old Mailing List's Archives `_ Bounties and Fundraisers ------------------------ From b1ba7a84f0f06ab2ce3be0cd413cde2dfc68284e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 19 Nov 2015 20:03:50 +0100 Subject: [PATCH 083/321] --keep-tag-files: fix file status, fix multiple tag files in one directory, fixes #432 --- borg/archiver.py | 9 ++++++--- borg/helpers.py | 13 +++++++------ borg/testsuite/archiver.py | 28 +++++++++++++++++++--------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index f7090b0c..f9ad44f6 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -210,11 +210,14 @@ class Archiver: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): - tag_path = dir_is_tagged(path, exclude_caches, exclude_if_present) - if tag_path: + tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present) + if tag_paths: if keep_tag_files: archive.process_dir(path, st) - archive.process_file(tag_path, st, cache) + for tag_path in tag_paths: + self._process(archive, cache, excludes, exclude_caches, exclude_if_present, + keep_tag_files, skip_inodes, tag_path, restrict_dev, + read_special=read_special, dry_run=dry_run) return if not dry_run: status = archive.process_dir(path, st) diff --git a/borg/helpers.py b/borg/helpers.py index e20aca6e..b4fbb7e5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -434,18 +434,19 @@ def dir_is_cachedir(path): def dir_is_tagged(path, exclude_caches, exclude_if_present): """Determines whether the specified path is excluded by being a cache - directory or containing the user-specified tag file. Returns the - path of the tag file (either CACHEDIR.TAG or the matching - user-specified file) + directory or containing user-specified tag files. Returns a list of the + paths of the tag files (either CACHEDIR.TAG or the matching + user-specified files). """ + tag_paths = [] if exclude_caches and dir_is_cachedir(path): - return os.path.join(path, 'CACHEDIR.TAG') + tag_paths.append(os.path.join(path, 'CACHEDIR.TAG')) if exclude_if_present is not None: for tag in exclude_if_present: tag_path = os.path.join(path, tag) if os.path.isfile(tag_path): - return tag_path - return None + tag_paths.append(tag_path) + return tag_paths def format_time(t): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index aaa75833..d61834d9 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -512,17 +512,27 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_exclude_keep_tagged(self): self.cmd('init', self.repository_location) - self.create_regular_file('file1', size=1024 * 80) - self.create_regular_file('tagged1/.NOBACKUP') - self.create_regular_file('tagged1/file2', size=1024 * 80) - self.create_regular_file('tagged2/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') - self.create_regular_file('tagged2/file3', size=1024 * 80) - self.cmd('create', '--exclude-if-present', '.NOBACKUP', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input') + self.create_regular_file('file0', size=1024) + self.create_regular_file('tagged1/.NOBACKUP1') + self.create_regular_file('tagged1/file1', size=1024) + self.create_regular_file('tagged2/.NOBACKUP2') + self.create_regular_file('tagged2/file2', size=1024) + self.create_regular_file('tagged3/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') + self.create_regular_file('tagged3/file3', size=1024) + self.create_regular_file('taggedall/.NOBACKUP1') + self.create_regular_file('taggedall/.NOBACKUP2') + self.create_regular_file('taggedall/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') + self.create_regular_file('taggedall/file4', size=1024) + self.cmd('create', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', + '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input') with changedir('output'): self.cmd('extract', self.repository_location + '::test') - self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'tagged1', 'tagged2']) - self.assert_equal(sorted(os.listdir('output/input/tagged1')), ['.NOBACKUP']) - self.assert_equal(sorted(os.listdir('output/input/tagged2')), ['CACHEDIR.TAG']) + self.assert_equal(sorted(os.listdir('output/input')), ['file0', 'tagged1', 'tagged2', 'tagged3', 'taggedall']) + self.assert_equal(os.listdir('output/input/tagged1'), ['.NOBACKUP1']) + self.assert_equal(os.listdir('output/input/tagged2'), ['.NOBACKUP2']) + self.assert_equal(os.listdir('output/input/tagged3'), ['CACHEDIR.TAG']) + self.assert_equal(sorted(os.listdir('output/input/taggedall')), + ['.NOBACKUP1', '.NOBACKUP2', 'CACHEDIR.TAG', ]) def test_path_normalization(self): self.cmd('init', self.repository_location) From 67c85734ba25588e6984668aa203d6abf653c55a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 01:12:31 +0100 Subject: [PATCH 084/321] upgrade OS X FUSE to 3.0.9, update release todo docs --- Vagrantfile | 4 ++-- docs/development.rst | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index dbb73f37..b7470591 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -55,9 +55,9 @@ end def packages_darwin return <<-EOF # get osxfuse 3.0.x pre-release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.0.5/osxfuse-3.0.5.dmg >osxfuse.dmg + curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.0.9/osxfuse-3.0.9.dmg >osxfuse.dmg MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.0.5.pkg" -target / + && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for OS X 3.0.9.pkg" -target / sudo chown -R vagrant /usr/local # brew must be able to create stuff here ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew update diff --git a/docs/development.rst b/docs/development.rst index def08b2d..753f3d55 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -170,4 +170,7 @@ Checklist: - create a Github release, include: * standalone binaries (see above for how to create them) + + for OS X, document the OS X Fuse version in the README of the binaries. + OS X FUSE uses a kernel extension that needs to be compatible with the + code contained in the binary. * a link to ``CHANGES.rst`` From 57ffa4d648b20527d768e1706a0d738a000632ef Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 15:49:53 +0100 Subject: [PATCH 085/321] more precise binary installation steps --- docs/installation.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index c6eb149b..e83f9e7f 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -62,10 +62,15 @@ on the releases_ page for the following platforms: * **Mac OS X**: 10.10 (unknown whether it works for older releases) * **FreeBSD**: 10.2 (unknown whether it works for older releases) -These binaries work without requiring specific installation steps. Just drop -them into a directory in your ``PATH`` and then you can run ``borg``. If a new -version is released, you will have to manually download it and replace the old -version. +To install such a binary, just drop it into a directory in your ``PATH``, +make borg readable and executable for its users and then you can run ``borg``:: + + sudo cp borg-linux64 /usr/local/bin/borg + sudo chown root:root /usr/local/bin/borg + sudo chmod 755 /usr/local/bin/borg + +If a new version is released, you will have to manually download it and replace +the old version using the same steps as shown above. .. _pyinstaller: http://www.pyinstaller.org .. _releases: https://github.com/borgbackup/borg/releases From 7a08368b650d819474bb891187e151aa1134e735 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 17:56:35 +0100 Subject: [PATCH 086/321] fix html_theme_path overriding previous definition of it --- docs/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 237f21ad..0283b7c2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -106,7 +106,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally #html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['_themes'] +#html_theme_path = ['_themes'] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". From 87bb88670584f9a5abeb13c8bc96a72b73519978 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 18:51:11 +0100 Subject: [PATCH 087/321] rtd theme adjustment: borg darkness for the upper left corner --- docs/borg_theme/css/borg.css | 18 ++++++++++++++++++ docs/conf.py | 3 ++- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 docs/borg_theme/css/borg.css diff --git a/docs/borg_theme/css/borg.css b/docs/borg_theme/css/borg.css new file mode 100644 index 00000000..6f4a7f41 --- /dev/null +++ b/docs/borg_theme/css/borg.css @@ -0,0 +1,18 @@ +@import url("theme.css"); + +/* The Return of the Borg. + * + * Have a bit green and grey and darkness (and if only in the upper left corner). + */ + +.wy-side-nav-search { + background-color: black; +} + +.wy-side-nav-search > a { + color: rgba(255, 255, 255, 0.5); +} + +.wy-side-nav-search > div.version { + color: rgba(255, 255, 255, 0.5); +} diff --git a/docs/conf.py b/docs/conf.py index 0283b7c2..8f7868b3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -127,7 +127,8 @@ html_favicon = '_static/favicon.ico' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -#html_static_path = ['_static'] +html_static_path = ['borg_theme'] +html_style = 'css/borg.css' # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From 077bcd0cdec7f54d8467749f719864f615837dd9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 19:06:40 +0100 Subject: [PATCH 088/321] logo: have a separate, bigger logo (do not reuse the favicon) currently just scaled up / converted to png from the favicon.ico. --- docs/_static/logo.png | Bin 0 -> 7031 bytes docs/conf.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 docs/_static/logo.png diff --git a/docs/_static/logo.png b/docs/_static/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..1f05c915bd425eea2ffca89962e89aee72663958 GIT binary patch literal 7031 zcmZvh2T&8vx9>w2F*HHxolrwlkRqW-6$kLm5y8r+HsHvf%f7>75)=zgx zZ`&O)U-H|I)LKhj1#t6k$!jl8zV(p5*D&z}04VPNTL}PJ*$lT%GA~UXRkB42I$A=u z7qRn!001+wri#)_znN`wCr@T|1PmAg@G}^*(yDKROdhl3X2Rt=aEpt&x$yZ0c?3AY zd%8tqkD4k(IzZtNv!PORt`Z*77f5qw7tqowh&5zOO{E9iu@YmbqH~@gAq}UQwG?Bh z0saren)x0ncVoAu*nccNJs!|BU}V^uJ>NFZ8F6O5Bsh zgiwO>c46fXwV{UcY-{BsrUSd%Dba6TzNF}q2%*0=GhP)?ZXyWr1n>bq0kQy61oi&N z{7EmqJn`z;71$7g*O-d6m|1}Jq{cE3@C|?ioYHl<(z>IJGZZD#$82cNNkSq4Lx4HJ zE;u9sP?2={PuMEuY1zR=FTjVOjbN9y%X0y9yVm3>EuLd}z#`z(7k~u3BKtMWVYQU_ z;hrT`7ZCstV0mAAstX7v{i*|$6IQ}8nR-$#fkPYsY@XJ#davg!#SF4ve^upLBz(`_ zr0|@yGi<`UM46aH_wU12< zdHYG=m-r6+$6Rc@-qR&sRGMZHaOqhu-hy;D;+^6ov0Kv+>VbYt(yHw{Vqhsz6T zRa3I~9*A*J{iP0}c@BbUc74>1|9fPhh4}c1wkLDdqZ%T0Q;bAv5O3zPCd!A8aPel} z8CP&Ip}7jw7O9Nea7);XJaGjuRFw9!bCJUtTw1?{`!C1txk^=D3}<2vWmw% z#<-x(+(w0io$Q;t;qsfqX=iNuaJCO1O61zo!HRk8_#zfQ(T_zhKB#MR_;-xC{G2U9 zWLe`v+Agd_rYHz8F=6jN_qc$Cj%!v)VMB*b`Zf(X@J*Cqa&_x`(7oWz~lN%Sk`(1S~YBFyw~9r0_=9plxOy1t8Hgn0(_ zB>9U4@DcdNf;DYKUOIWuoQRzp9cD`Qo=lGx-NNMu{*r7~*Y4yL9=m9LTN^O5E|Qmcq~K??tv{ zO_(1guFI?)Y&42;KG#NEjMIM^*(sdm*Qzk&(D7Loq#Sc8po>5J92k`CNl4#}W&avk1H+=3$e(5@pY+p&ECMbyeDmRI<;jk6a3ji{ z-Ra5GSQ8_~Jks*FJ2HS>xe)b~ZAF+5YgcD&!n?Alid^?<-?u9#a3cvd`uHGYpCpX6{z(>mKc@x{U}PonB$#WzlhJ-mcp> zCL48{h4+=QykKK*=?>_yW0sP!oRHbYA7lwVcfmST30PB|-g`zHsA!$D`WQpBwy-fD z!2W&-=s|7jGnP}s%I!R!wxz4NM^6|vv0y9596!8?RyyW=YOqpJg4I*;QN|e{8mm)6 zGq!yJmlXLxJ!XwX>cS}4cZkB81J-$e%Z%Qdk49c+h431c{vPeXzkkJ>Wy$WI7=$^w zV4;(BYu|z}%U$o0HsFGH*CWT40!T<F0(V351k zu{iL~u{}qGA5mque>;6@u5Dl6%w(!v?P>Qt$_dMKE~1cq+miqekDiGsju%cB8y;m_F-fa=%n% z`6tO{DIjar31_|aa{WZhCf)o=#dkWlUlFrkHjf`tPOzfn`bS8eX~pM_2Oqh|{n72N zl!45F`nm1TbRV0~C1UDv#QxEK z!?4s^t>>;)G9G26Q|cL3s-K+czw|)P?(Gjz~Dvyz*gQiiCrSE`s8ZTlTTwTV?0S*3np*76gYb(VV8WJzI; z6UDPxU;gl-yDCSt^>g*L6duqZ|6_6Dt)Tr1p?j~;=mEy%qnoRJ`?EluJ&EM*JJ(nNP%hhj@$9{)!yP)5fS3E5ed;RzyeQ4UrG;k45EU@|Mxb+7d%jNl zmzXF4FX2-o@uaN%*+}$qUvzR)=;J8fjZ{Q>tkelh<#yJhzioOdT|qk2T6;A}colx5 zVfJib-ws`+&ThJDxPs3)`8Z`qKJCCA0JSWR^%pss^C~i2x_on2W}$EsMDubo0^_W; zG%&F)_=|@~rM6sFki$S|DQsg+%r)(Xne2pHMU$pA1NL*`*CP|nDrMudWSs=Y&pz_3 zfiw1)w=p3gmO!+MKOjj|8-AO^v%<`N7z%O_Q{Kjik#W2@LNMBix0Ij(=Z< z-MW=EE%h~j&KqNu3OOPO|9dx9AREuA@p9+9YQ4&Vq)wAAMaE*soiCZM8YPO>wT`LQ))-InacW4S?$J+(4Zy!NR?W=`$pCpGI4SM{ z8rUb6w1|21lzmBAZ6pqGA`bJeb{>Rh2yvCq9|f7tqIs|%2Bi1acWfj+dQY27kbB&5 z>m%S;y#jVsUz(JabS%MiWdLrwLEl(BrL;MH^9J^Hpp+6$Z#gaGa!E=X47p@ZE#=5k z(jA+?^hp*Xj%nDwebW53j@j&|Oz0WfX_%R^4@C9x%kq0fD6JURHHd5JUh{G$3u0iu z zHVb-FPFYh9&HIho`P$x!n|w1BFT(fvA|*iM`{tmi|37;YVaC+>({h19|If9p<}Z98 zGM`dLA%j;TuBPg31N_@ta^j(CLq1MF`F46-7Yru0)nnA&B7<9_9CHpjH%xiP=_fIA zKzg8S2^3RjrSuu%rm=OSnnDo>?+1U*r&n=#x9dhE=~t)SlqJ!rD?=Sbvg_NyZY}Y7 zCnGeEp1vPQW2ZT9XIdw9uSv@ac$~V>p4!0d#MB&ZkJTT|9^rB-lkO)a!-9e>42MlH z;(pJH*_omhU!bP)aZ(iMAwKmNHf(~(YkH)N*Nk`giG^yLcKU>fJm>K^>MI)imc!!5 zo4%z??2ks8YOSECK<}TUw9+HQ+|w57Jj0@Z>nQ%4(>T7@6_N{gLB7Fuc4f_4rZ#Ig ztXvD@kKT816V%_Q&DYdF$#Ha{aW(J-b+_EFD0}Wg79Z;soGVLE0wJr+$RIMCBYF5Y zqit?bQg(;yK{|?m|8`=%S#{dpRRiwd%w1S)%nS3v2)CK6B$5g%bAzp-;vzC71Y~|( zz1?v9ToHda#hat(k7(ePwsog!jW9C5c~j|o$UOs4djYvng{5-Nx*!}BWaRPuw~KvT zjYKV`rMlXDwCa+lp>hd59GKpEn3Y#01mPF6aqu?NIV(~>EWSyA?64sw@4!mKtq3X+ zbWQi`nnG8d!82YK&ol1)poPIQCFe&T#}1uU|FH(Pa*z?s51%0d$`W&i3Cjxm2EREQ zq|Mpb7MF_9)f`Zde}kAm?wpm3Qg^0Jn&eksH_F*eliFx7F?n`o6Z0d5{QUw`ThUDM ztcfqzjTM?_v%}0Jk{)Vn1syDnV&{3~Vz*W^?{_uR;V1`QZ5@ zwZLlRl1*eha!buWJOWSbj7T(4;xSRl|I^B!UF(p67$ZVoQ|s|GtDo79D|jYor%5i@ zn8m7)so^lZ-)v_!a`6e5{zTU^dJ=V7sahtVQ-D;JDIqb?sYG*K1S4<5z#83<%|L;5 zLAQ;nH#P4{g+Ap6q)*!4RPv5U$9^1ejn9AC|49ZzjYD3|$0?4`}`OcEd!ob zL5>)I+8;(p(MCXBct09Kgm@i>PbI8}*|?97D9~0^$==HcMPyo)4bLoAz{}a&wMy#T z$8-eTu!0?de4ZpCl*v;_oRB7v51y0;e7yiura%=%yDnnE8H5123}IibK=pVT zK8JMBv?7d<>zJXy6GybixZ#h z1{Mt*w95VTQq@{?D1+R3*!Qn65RYwifLPB`|qZE@J;G@@DtYQY|^A+A(k; zwx|FP0{8Yw&YNp#B85hKu&a|$?$&~!6IXF>s_>>`R2mdvy?I3ZXXBuz5LEv%ECs*X zZ!|Lg(iUup{IX`>rWwU+|(S0Zf~we(+ZvFUy?Mb38{AxAB2h^ z>(20|rX>eB6|1lA+X0EeRbl)7R|p{SyBhye8*dfbh2<+Aa6|f^yBm$@%qix;Yu(d( z7#0rwLj8FEBvdA=_Jd_iUE6v&gRElU#sZ+O)VEP1QTwN_0UF9si^d0QOgGTgsKc?- zn-af0r3Tjx4zwRY<;XPe>shHYM(A(o4b4Djhc>bjLPvON^^_g3 z=0_Yek84NQOSC@a$I58JTYd()?!k2Xdn>D@_&kQIe}#<&b%bu9qN~pi#H;{X1fM|5 zIf`0MuL?vo!<|Br??1slIXX;gAzW~C)iF(|iVQ1P-ottW;X_XEkqK`~F^-|Q9WD`E zQ+SEl4jAe&yPG})e5Ape*j1~v%$Q%>tKQVq&)bxbN^@;?;k7L$7vU{;E0G=st1eUG z-7D(-VArsx>~WP(Y~jQH>VpwZ>-V&dVr6UZUmY#Xi&w*<9b1Qk&V&)2Mx+72bapEDI^z!!GhF|d6PE{#^-~E9I(^S zzY9c#E?^k>TkfKA)qHGvXUYV&!A82DSBgr76|85hp0Udajd7JhP9s3n^Io4a`R1 zcXvsy(Lh69kl%Qcsc&WSVlGUz`!;!C?UeH}%Y7DK{0x;D{>s)mlFgONUV|uEm69;w zub0aQqK6b!SS;GbEZ=7C0x6_(74iB!JuFKWQ=jbc(BLgbd|X+V8VoKM-gVshK-zpB ze|Gig4K0c8XOFk_UuGPg&K3-pKqtF+efCG0gPgZnWy^ znsi0co0u(1zkp3CwXO1AaX3geOzVQpDrOj^ji8a?&$ZpeJ9QIpR9=8h$#=orP{Ut;tUEJn1qID95&7&2Qk&=`ao-{B<}Wqr#V6Sgb~(4NVu^s34$r4)?x;Nn45b9Cv47#rtL z#g`W?TZvB_N(&5*sd*!^U(B|xSk5VC6G#bfnrc;U5@i)8&>&0v;m$&d(M~6{$>cni zpN*Utn`VqWdZ<#H1AEqk2mAace+PDW78guRy4VFN3|a-X?z8Ut`}*TUI={K^RqXS) zKs~k+WR^|@B2{(F$7@{#z9*#ONaM>(qdK&7t`co^Vvm6auqwG1zBMDR%7>GXKcSnp+lL>w5ST5DjR!1P{|L??R3HC2Axp>0Rb z$GLS?a(;-~Q(sE{P;9m!mS$P7T-53ONs`&AMfy-`VH1HuXZ7Wduoc>SO?!KkmdT@l zP|=MViA;U)730wmvE)E97wVt2M&|4h6cWRm2!lah!XRj|oS5u{oL>5>fnGMD}FxY$29sPU!DU46^v6SVo4o6EOkpE*!qsA!79`Ci-f0ydTgpxYqe~Wdc4Ua@=}7@ z{<);fYa?WEiD@>@*U(v2x;f!H+>L)e^gMKa?z#BS+1#@;#PCk-8;gSgIKN_Jg7oAK zpYyi43c=%B81czV8B@E>n7{TjX}`5sLo_`la*CVB0k^r~YnpUUA621wMY^~7<(7GH zhEEZxVARRrga!eDzP>A|lD$b^_K12fA$gluikXXUXcy;yWtFwp3=ro6S6Ij*Z`Srs z5vIH$G^Vh14z^Wwb_LIR-5Bgs>{diG`(FOHQVQXv!sQu2uW*hP)p|hqGFhhBh4PSE zwYl(kL3jLg>{gaw?3eZaj$U|`K;`ydY$QFkDO!hpC70vIN;|HJ^dLz5&n4X~U z676)Hxk6>`$6F;O=SVG?4c?cg6B=a0hd%S31rLS9Z}*;F>`OeR)S$>0G3dBnaC_uU z`D*b00f+vJ>HG&A`oFm7Kj6@R;iCVY68(4X^hvAdhA0v)Pi$f|6L*Wq0Gg^$l`3V+ Gu>S)Hg{9#D literal 0 HcmV?d00001 diff --git a/docs/conf.py b/docs/conf.py index 8f7868b3..e5ce40f8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -117,7 +117,7 @@ if not on_rtd: # only import and set the theme if we're building docs locally # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = '_static/favicon.ico' +html_logo = '_static/logo.png' # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 From 42e9a77f5bfc6c256dbc3763be4154080e86d290 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 20 Nov 2015 19:32:04 +0100 Subject: [PATCH 089/321] style changes of previous changeset did not work on rtd, try 2 --- docs/conf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index e5ce40f8..9c862cc5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -99,6 +99,15 @@ if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme html_theme = 'sphinx_rtd_theme' html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] + html_style = 'css/borg.css' +else: + html_context = { + 'css_files': [ + 'https://media.readthedocs.org/css/sphinx_rtd_theme.css', + 'https://media.readthedocs.org/css/readthedocs-doc-embed.css', + '_static/css/borg.css', + ], + } # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -128,7 +137,6 @@ html_favicon = '_static/favicon.ico' # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['borg_theme'] -html_style = 'css/borg.css' # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. From a40729f4f3c37119b3af5dd4b49e0c1973fc40cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 20 Nov 2015 14:56:18 -0500 Subject: [PATCH 090/321] --progress option was backwards adds unit tests and ensures we detect --progress correctly in all cases --- borg/archiver.py | 2 +- borg/testsuite/archiver.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 12de588e..479d5a85 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -775,7 +775,7 @@ class Archiver: """ def __call__(self, parser, ns, values, option): """set the given flag to true unless ``--no`` is passed""" - setattr(ns, self.dest, option.startswith('--no-')) + setattr(ns, self.dest, not option.startswith('--no-')) subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__, diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d61834d9..48b9da0c 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -669,6 +669,28 @@ class ArchiverTestCase(ArchiverTestCaseBase): manifest, key = Manifest.load(repository) self.assert_equal(len(manifest.archives), 0) + def test_progress(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', self.repository_location) + # without a terminal, no progress expected + output = self.cmd('create', self.repository_location + '::test1', 'input', fork=False) + self.assert_not_in("\r", output) + # with a terminal, progress expected + output = self.cmd('create', self.repository_location + '::test2', 'input', fork=True) + self.assert_in("\r", output) + # without a terminal, progress forced on + output = self.cmd('create', '--progress', self.repository_location + '::test3', 'input', fork=False) + self.assert_in("\r", output) + # with a terminal, progress forced on + output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input', fork=True) + self.assert_in("\r", output) + # without a termainl, progress forced off + output = self.cmd('create', '--no-progress', self.repository_location + '::test5', 'input', fork=False) + self.assert_not_in("\r", output) + # with a termainl, progress forced off + output = self.cmd('create', '--no-progress', self.repository_location + '::test6', 'input', fork=True) + self.assert_not_in("\r", output) + def test_cmdline_compatibility(self): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) From a6f8436ceb723e5d86b0abe47d86f0d1a1e687d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 20 Nov 2015 15:01:29 -0500 Subject: [PATCH 091/321] move toggle action to beginning of class so it can be reused --- borg/archiver.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 479d5a85..3e334ec5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -35,6 +35,20 @@ from .remote import RepositoryServer, RemoteRepository has_lchflags = hasattr(os, 'lchflags') +class ToggleAction(argparse.Action): + """argparse action to handle "toggle" flags easily + + toggle flags are in the form of ``--foo``, ``--no-foo``. + + the ``--no-foo`` argument still needs to be passed to the + ``add_argument()`` call, but it simplifies the ``--no`` + detection. + """ + def __call__(self, parser, ns, values, option): + """set the given flag to true unless ``--no`` is passed""" + setattr(ns, self.dest, not option.startswith('--no-')) + + class Archiver: def __init__(self, verbose=False): @@ -764,19 +778,6 @@ class Archiver: See the output of the "borg help patterns" command for more help on exclude patterns. """) - class ToggleAction(argparse.Action): - """argparse action to handle "toggle" flags easily - - toggle flags are in the form of ``--foo``, ``--no-foo``. - - the ``--no-foo`` argument still needs to be passed to the - ``add_argument()`` call, but it simplifies the ``--no`` - detection. - """ - def __call__(self, parser, ns, values, option): - """set the given flag to true unless ``--no`` is passed""" - setattr(ns, self.dest, not option.startswith('--no-')) - subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog, From c4dae52ca4b42b605362a5b52fa5f4ef220c936d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Nov 2015 20:17:54 +0100 Subject: [PATCH 092/321] configure logging via env var, use LazyLogger logging.raiseExceptions not needed any more for py >= 3.2 --- borg/archiver.py | 2 +- borg/logger.py | 117 +++++++++++++++++++++++++++++++++------ borg/testsuite/logger.py | 17 +++++- docs/usage.rst | 4 ++ 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 12de588e..73fc5098 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1155,7 +1155,7 @@ class Archiver: self.verbose = args.verbose RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask - setup_logging() + setup_logging() # do not use loggers before this! check_extension_modules() keys_dir = get_keys_dir() if not os.path.exists(keys_dir): diff --git a/borg/logger.py b/borg/logger.py index 2d4b49bc..07bb3770 100644 --- a/borg/logger.py +++ b/borg/logger.py @@ -32,28 +32,70 @@ The way to use this is as follows: import inspect import logging - -# make it easy for PyInstaller (it does not discover the dependency on this -# module automatically, because it is lazy-loaded by logging, see #218): import logging.config +import logging.handlers # needed for handlers defined there being configurable in logging.conf file +import os +import warnings + +configured = False + +# use something like this to ignore warnings: +# warnings.filterwarnings('ignore', r'... regex for warning message to ignore ...') -def setup_logging(stream=None): +def _log_warning(message, category, filename, lineno, file=None, line=None): + # for warnings, we just want to use the logging system, not stderr or other files + msg = "{0}:{1}: {2}: {3}".format(filename, lineno, category.__name__, message) + logger = create_logger(__name__) + # Note: the warning will look like coming from here, + # but msg contains info about where it really comes from + logger.warning(msg) + + +def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF'): """setup logging module according to the arguments provided - this sets up a stream handler logger on stderr (by default, if no + if conf_fname is given (or the config file name can be determined via + the env_var, if given): load this logging configuration. + + otherwise, set up a stream handler logger on stderr (by default, if no stream is provided). """ - logging.raiseExceptions = False - l = logging.getLogger('') - sh = logging.StreamHandler(stream) - # other formatters will probably want this, but let's remove - # clutter on stderr + global configured + err_msg = None + if env_var: + conf_fname = os.environ.get(env_var, conf_fname) + if conf_fname: + try: + conf_fname = os.path.abspath(conf_fname) + # we open the conf file here to be able to give a reasonable + # error message in case of failure (if we give the filename to + # fileConfig(), it silently ignores unreadable files and gives + # unhelpful error msgs like "No section: 'formatters'"): + with open(conf_fname) as f: + logging.config.fileConfig(f) + configured = True + logger = logging.getLogger(__name__) + logger.debug('using logging configuration read from "{0}"'.format(conf_fname)) + warnings.showwarning = _log_warning + return None + except Exception as err: # XXX be more precise + err_msg = str(err) + # if we did not / not successfully load a logging configuration, fallback to this: + logger = logging.getLogger('') + handler = logging.StreamHandler(stream) + # other formatters will probably want this, but let's remove clutter on stderr # example: - # sh.setFormatter(logging.Formatter('%(name)s: %(message)s')) - l.addHandler(sh) - l.setLevel(logging.INFO) - return sh + # handler.setFormatter(logging.Formatter('%(name)s: %(message)s')) + logger.addHandler(handler) + logger.setLevel(logging.INFO) + configured = True + logger = logging.getLogger(__name__) + if err_msg: + logger.warning('setup_logging for "{0}" failed with "{1}".'.format(conf_fname, err_msg)) + logger.debug('using builtin fallback logging configuration') + warnings.showwarning = _log_warning + return handler def find_parent_module(): @@ -76,7 +118,7 @@ def find_parent_module(): def create_logger(name=None): - """create a Logger object with the proper path, which is returned by + """lazily create a Logger object with the proper path, which is returned by find_parent_module() by default, or is provided via the commandline this is really a shortcut for: @@ -84,5 +126,48 @@ def create_logger(name=None): logger = logging.getLogger(__name__) we use it to avoid errors and provide a more standard API. + + We must create the logger lazily, because this is usually called from + module level (and thus executed at import time - BEFORE setup_logging() + was called). By doing it lazily we can do the setup first, we just have to + be careful not to call any logger methods before the setup_logging() call. + If you try, you'll get an exception. """ - return logging.getLogger(name or find_parent_module()) + class LazyLogger: + def __init__(self, name=None): + self.__name = name or find_parent_module() + self.__real_logger = None + + @property + def __logger(self): + if self.__real_logger is None: + if not configured: + raise Exception("tried to call a logger before setup_logging() was called") + self.__real_logger = logging.getLogger(self.__name) + return self.__real_logger + + def setLevel(self, *args, **kw): + return self.__logger.setLevel(*args, **kw) + + def log(self, *args, **kw): + return self.__logger.log(*args, **kw) + + def exception(self, *args, **kw): + return self.__logger.exception(*args, **kw) + + def debug(self, *args, **kw): + return self.__logger.debug(*args, **kw) + + def info(self, *args, **kw): + return self.__logger.info(*args, **kw) + + def warning(self, *args, **kw): + return self.__logger.warning(*args, **kw) + + def error(self, *args, **kw): + return self.__logger.error(*args, **kw) + + def critical(self, *args, **kw): + return self.__logger.critical(*args, **kw) + + return LazyLogger(name) diff --git a/borg/testsuite/logger.py b/borg/testsuite/logger.py index ff8b0a03..b6dc2965 100644 --- a/borg/testsuite/logger.py +++ b/borg/testsuite/logger.py @@ -10,7 +10,7 @@ logger = create_logger() @pytest.fixture() def io_logger(): io = StringIO() - handler = setup_logging(io) + handler = setup_logging(stream=io, env_var=None) handler.setFormatter(logging.Formatter('%(name)s: %(message)s')) logger.setLevel(logging.DEBUG) return io @@ -37,3 +37,18 @@ def test_multiple_loggers(io_logger): def test_parent_module(): assert find_parent_module() == __name__ + + +def test_lazy_logger(): + # just calling all the methods of the proxy + logger.setLevel(logging.DEBUG) + logger.debug("debug") + logger.info("info") + logger.warning("warning") + logger.error("error") + logger.critical("critical") + logger.log(logging.INFO, "info") + try: + raise Exception + except Exception: + logger.exception("exception") diff --git a/docs/usage.rst b/docs/usage.rst index 7a06ec8f..d7515f67 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -47,6 +47,8 @@ General: can either leave it away or abbreviate as `::`, if a positional parameter is required. BORG_PASSPHRASE When set, use the value to answer the passphrase question for encrypted repositories. + BORG_LOGGING_CONF + When set, use the given filename as INI_-style logging configuration. BORG_RSH When set, use this command instead of ``ssh``. TMPDIR @@ -81,6 +83,8 @@ Please note: (e.g. mode 600, root:root). +.. _INI: https://docs.python.org/3.2/library/logging.config.html#configuration-file-format + Resource Usage ~~~~~~~~~~~~~~ From 25140e8c8218fe19ef9ffc8029e3f53516b2d546 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 02:09:16 +0100 Subject: [PATCH 093/321] add --log-level to set the level of the builtin logging configuration, fixes #426 --- borg/archiver.py | 5 ++++- borg/logger.py | 4 ++-- docs/usage.rst | 19 ++++++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 73fc5098..7a2b5db4 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -646,6 +646,9 @@ class Archiver: common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, help='verbose output') + common_parser.add_argument('--log-level', dest='log_level', default='info', metavar='LEVEL', + choices=('debug', 'info', 'warning', 'error', 'critical'), + help='set the log level to LEVEL, default: %(default)s)') common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, help='show/log the return code (rc)') common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', @@ -1155,7 +1158,7 @@ class Archiver: self.verbose = args.verbose RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask - setup_logging() # do not use loggers before this! + setup_logging(level=args.log_level) # do not use loggers before this! check_extension_modules() keys_dir = get_keys_dir() if not os.path.exists(keys_dir): diff --git a/borg/logger.py b/borg/logger.py index 07bb3770..a40f676c 100644 --- a/borg/logger.py +++ b/borg/logger.py @@ -52,7 +52,7 @@ def _log_warning(message, category, filename, lineno, file=None, line=None): logger.warning(msg) -def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF'): +def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info'): """setup logging module according to the arguments provided if conf_fname is given (or the config file name can be determined via @@ -88,7 +88,7 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF'): # example: # handler.setFormatter(logging.Formatter('%(name)s: %(message)s')) logger.addHandler(handler) - logger.setLevel(logging.INFO) + logger.setLevel(level.upper()) configured = True logger = logging.getLogger(__name__) if err_msg: diff --git a/docs/usage.rst b/docs/usage.rst index d7515f67..5bba8390 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -11,12 +11,21 @@ command in detail. General ------- -Quiet by default -~~~~~~~~~~~~~~~~ +Type of log output +~~~~~~~~~~~~~~~~~~ -Like most UNIX commands |project_name| is quiet by default but the ``-v`` or -``--verbose`` option can be used to get the program to output more status -messages as it is processing. +You can set the log level of the builtin logging configuration using the +--log-level option. + +Supported levels: ``debug``, ``info``, ``warning``, ``error``, ``critical``. + +All log messages created with at least the given level will be output. + +Amount of informational output +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For some subcommands, using the ``-v`` or ``--verbose`` option will give you +more informational output (at ``info`` level). Return codes ~~~~~~~~~~~~ From 6abf7621c192b5c870b2da37a3ac1d089311749d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 02:22:26 +0100 Subject: [PATCH 094/321] remove rarely used print_status method --- borg/archiver.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 7a2b5db4..fb9d9b24 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -64,9 +64,6 @@ class Archiver: msg = args and msg % args or msg logger.info(msg) - def print_status(self, status, path): - self.print_info("%1s %s", status, remove_surrogates(path)) - def do_serve(self, args): """Start in server mode. This command is usually not used manually. """ @@ -154,7 +151,7 @@ class Archiver: self.print_warning('%s: %s', path, e) else: status = '-' - self.print_status(status, path) + self.print_info("%1s %s", status, remove_surrogates(path)) continue path = os.path.normpath(path) if args.one_file_system: @@ -255,7 +252,7 @@ class Archiver: status = '-' # dry run, item was not backed up # output ALL the stuff - it can be easily filtered using grep. # even stuff considered unchanged might be interesting. - self.print_status(status, path) + self.print_info("%1s %s", status, remove_surrogates(path)) def do_extract(self, args): """Extract archive contents""" From b3b4db427cce182400c9f7b94a15018eaa30b5c4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 02:26:50 +0100 Subject: [PATCH 095/321] rename print_info to print_verbose better name as it is only outputting if verbose flag is set. --- borg/archiver.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index fb9d9b24..b65a7630 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -59,7 +59,7 @@ class Archiver: self.exit_code = EXIT_WARNING # we do not terminate here, so it is a warning logger.warning(msg) - def print_info(self, msg, *args): + def print_verbose(self, msg, *args): if self.verbose: msg = args and msg % args or msg logger.info(msg) @@ -151,7 +151,7 @@ class Archiver: self.print_warning('%s: %s', path, e) else: status = '-' - self.print_info("%1s %s", status, remove_surrogates(path)) + self.print_verbose("%1s %s", status, remove_surrogates(path)) continue path = os.path.normpath(path) if args.one_file_system: @@ -252,7 +252,7 @@ class Archiver: status = '-' # dry run, item was not backed up # output ALL the stuff - it can be easily filtered using grep. # even stuff considered unchanged might be interesting. - self.print_info("%1s %s", status, remove_surrogates(path)) + self.print_verbose("%1s %s", status, remove_surrogates(path)) def do_extract(self, args): """Extract archive contents""" @@ -280,7 +280,7 @@ class Archiver: if not args.dry_run: while dirs and not item[b'path'].startswith(dirs[-1][b'path']): archive.extract_item(dirs.pop(-1), stdout=stdout) - self.print_info(remove_surrogates(orig_path)) + self.print_verbose(remove_surrogates(orig_path)) try: if dry_run: archive.extract_item(item, dry_run=True) @@ -366,7 +366,7 @@ class Archiver: else: archive = None operations = FuseOperations(key, repository, manifest, archive) - self.print_info("Mounting filesystem") + self.print_verbose("Mounting filesystem") try: operations.mount(args.mountpoint, args.options, args.foreground) except RuntimeError: @@ -469,12 +469,12 @@ class Archiver: to_delete = [a for a in archives if a not in keep] stats = Statistics() for archive in keep: - self.print_info('Keeping archive: %s' % format_archive(archive)) + self.print_verbose('Keeping archive: %s' % format_archive(archive)) for archive in to_delete: if args.dry_run: - self.print_info('Would prune: %s' % format_archive(archive)) + self.print_verbose('Would prune: %s' % format_archive(archive)) else: - self.print_info('Pruning archive: %s' % format_archive(archive)) + self.print_verbose('Pruning archive: %s' % format_archive(archive)) Archive(repository, key, manifest, archive.name, cache).delete(stats) if to_delete and not args.dry_run: manifest.write() From f19e95fcf77cb533073031dafd2049478e231a9d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 15:34:51 +0100 Subject: [PATCH 096/321] implement --lock-wait, support timeout for UpgradableLock, fixes #210 also: simplify exceptions / exception handling --- borg/archiver.py | 20 ++++---- borg/cache.py | 9 ++-- borg/locking.py | 92 +++++++++++++++++------------------- borg/remote.py | 8 ++-- borg/repository.py | 12 ++--- borg/testsuite/locking.py | 16 ++++++- borg/testsuite/repository.py | 6 +-- 7 files changed, 88 insertions(+), 75 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index b65a7630..55563d3a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -37,15 +37,16 @@ has_lchflags = hasattr(os, 'lchflags') class Archiver: - def __init__(self, verbose=False): + def __init__(self, verbose=False, lock_wait=None): self.exit_code = EXIT_SUCCESS self.verbose = verbose + self.lock_wait = lock_wait def open_repository(self, location, create=False, exclusive=False): if location.proto == 'ssh': - repository = RemoteRepository(location, create=create) + repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait) else: - repository = Repository(location.path, create=create, exclusive=exclusive) + repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait) repository._location = location return repository @@ -119,7 +120,7 @@ class Archiver: compr_args = dict(buffer=COMPR_BUFFER) compr_args.update(args.compression) key.compressor = Compressor(**compr_args) - cache = Cache(repository, key, manifest, do_files=args.cache_files) + cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner, progress=args.progress, @@ -305,7 +306,7 @@ class Archiver: """Rename an existing archive""" repository = self.open_repository(args.archive, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest) + cache = Cache(repository, key, manifest, lock_wait=self.lock_wait) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) archive.rename(args.name) manifest.write() @@ -317,7 +318,7 @@ class Archiver: """Delete an existing repository or archive""" repository = self.open_repository(args.target, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files) + cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) if args.target.archive: archive = Archive(repository, key, manifest, args.target.archive, cache=cache) stats = Statistics() @@ -424,7 +425,7 @@ class Archiver: """Show archive details such as disk space used""" repository = self.open_repository(args.archive) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files) + cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) stats = archive.calc_stats(cache) print('Name:', archive.name) @@ -443,7 +444,7 @@ class Archiver: """Prune repository archives according to specified rules""" repository = self.open_repository(args.repository, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files) + cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0 and args.within is None: self.print_error('At least one of the "within", "keep-hourly", "keep-daily", "keep-weekly", ' @@ -646,6 +647,8 @@ class Archiver: common_parser.add_argument('--log-level', dest='log_level', default='info', metavar='LEVEL', choices=('debug', 'info', 'warning', 'error', 'critical'), help='set the log level to LEVEL, default: %(default)s)') + common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, + help='wait for the lock, but max. N seconds (default: %(default)d).') common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, help='show/log the return code (rc)') common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', @@ -1153,6 +1156,7 @@ class Archiver: def run(self, args): os.umask(args.umask) # early, before opening files self.verbose = args.verbose + self.lock_wait = args.lock_wait RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask setup_logging(level=args.log_level) # do not use loggers before this! diff --git a/borg/cache.py b/borg/cache.py index 89aedd4f..6116df49 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -32,7 +32,8 @@ class Cache: class EncryptionMethodMismatch(Error): """Repository encryption method changed since last access, refusing to continue""" - def __init__(self, repository, key, manifest, path=None, sync=True, do_files=False, warn_if_unencrypted=True): + def __init__(self, repository, key, manifest, path=None, sync=True, do_files=False, warn_if_unencrypted=True, + lock_wait=None): self.lock = None self.timestamp = None self.lock = None @@ -52,7 +53,7 @@ class Cache: env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): raise self.CacheInitAbortedError() self.create() - self.open() + self.open(lock_wait=lock_wait) # Warn user before sending data to a relocated repository if self.previous_location and self.previous_location != repository._location.canonical_path(): msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + @@ -136,10 +137,10 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" self.chunks = ChunkIndex.read(os.path.join(self.path, 'chunks').encode('utf-8')) self.files = None - def open(self): + def open(self, lock_wait=None): if not os.path.isdir(self.path): raise Exception('%s Does not look like a Borg cache' % self.path) - self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire() + self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=lock_wait).acquire() self.rollback() def close(self): diff --git a/borg/locking.py b/borg/locking.py index 159e3c57..d3f309cb 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -74,26 +74,32 @@ class TimeoutTimer: return False +class LockError(Error): + """Failed to acquire the lock {}.""" + + +class LockErrorT(ErrorWithTraceback): + """Failed to acquire the lock {}.""" + + +class LockTimeout(LockError): + """Failed to create/acquire the lock {} (timeout).""" + + +class LockFailed(LockErrorT): + """Failed to create/acquire the lock {} ({}).""" + + +class NotLocked(LockErrorT): + """Failed to release the lock {} (was not locked).""" + + +class NotMyLock(LockErrorT): + """Failed to release the lock {} (was/is locked, but not by me).""" + + class ExclusiveLock: """An exclusive Lock based on mkdir fs operation being atomic""" - class LockError(ErrorWithTraceback): - """Failed to acquire the lock {}.""" - - class LockTimeout(LockError): - """Failed to create/acquire the lock {} (timeout).""" - - class LockFailed(LockError): - """Failed to create/acquire the lock {} ({}).""" - - class UnlockError(ErrorWithTraceback): - """Failed to release the lock {}.""" - - class NotLocked(UnlockError): - """Failed to release the lock {} (was not locked).""" - - class NotMyLock(UnlockError): - """Failed to release the lock {} (was/is locked, but not by me).""" - def __init__(self, path, timeout=None, sleep=None, id=None): self.timeout = timeout self.sleep = sleep @@ -124,9 +130,9 @@ class ExclusiveLock: if self.by_me(): return self if timer.timed_out_or_sleep(): - raise self.LockTimeout(self.path) + raise LockTimeout(self.path) else: - raise self.LockFailed(self.path, str(err)) + raise LockFailed(self.path, str(err)) else: with open(self.unique_name, "wb"): pass @@ -134,9 +140,9 @@ class ExclusiveLock: def release(self): if not self.is_locked(): - raise self.NotLocked(self.path) + raise NotLocked(self.path) if not self.by_me(): - raise self.NotMyLock(self.path) + raise NotMyLock(self.path) os.unlink(self.unique_name) os.rmdir(self.path) @@ -215,23 +221,18 @@ class UpgradableLock: noone is allowed reading) and read access to a resource needs a shared lock (multiple readers are allowed). """ - class SharedLockFailed(ErrorWithTraceback): - """Failed to acquire shared lock [{}]""" - - class ExclusiveLockFailed(ErrorWithTraceback): - """Failed to acquire write lock [{}]""" - - def __init__(self, path, exclusive=False, sleep=None, id=None): + def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None): self.path = path self.is_exclusive = exclusive self.sleep = sleep + self.timeout = timeout self.id = id or get_id() # globally keeping track of shared and exclusive lockers: self._roster = LockRoster(path + '.roster', id=id) # an exclusive lock, used for: # - holding while doing roster queries / updates # - holding while the UpgradableLock itself is exclusive - self._lock = ExclusiveLock(path + '.exclusive', id=id) + self._lock = ExclusiveLock(path + '.exclusive', id=id, timeout=timeout) def __enter__(self): return self.acquire() @@ -246,25 +247,19 @@ class UpgradableLock: if exclusive is None: exclusive = self.is_exclusive sleep = sleep or self.sleep or 0.2 - try: - if exclusive: - self._wait_for_readers_finishing(remove, sleep) - self._roster.modify(EXCLUSIVE, ADD) - else: - with self._lock: - if remove is not None: - self._roster.modify(remove, REMOVE) - self._roster.modify(SHARED, ADD) - self.is_exclusive = exclusive - return self - except ExclusiveLock.LockError as err: - msg = str(err) - if exclusive: - raise self.ExclusiveLockFailed(msg) - else: - raise self.SharedLockFailed(msg) + if exclusive: + self._wait_for_readers_finishing(remove, sleep) + self._roster.modify(EXCLUSIVE, ADD) + else: + with self._lock: + if remove is not None: + self._roster.modify(remove, REMOVE) + self._roster.modify(SHARED, ADD) + self.is_exclusive = exclusive + return self def _wait_for_readers_finishing(self, remove, sleep): + timer = TimeoutTimer(self.timeout, sleep).start() while True: self._lock.acquire() if remove is not None: @@ -273,7 +268,8 @@ class UpgradableLock: if len(self._roster.get(SHARED)) == 0: return # we are the only one and we keep the lock! self._lock.release() - time.sleep(sleep) + if timer.timed_out_or_sleep(): + raise LockTimeout(self.path) def release(self): if self.is_exclusive: diff --git a/borg/remote.py b/borg/remote.py index df435ebc..1d5e9cab 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -97,7 +97,7 @@ class RepositoryServer: # pragma: no cover def negotiate(self, versions): return 1 - def open(self, path, create=False): + def open(self, path, create=False, lock_wait=None): path = os.fsdecode(path) if path.startswith('/~'): path = path[1:] @@ -108,7 +108,7 @@ class RepositoryServer: # pragma: no cover break else: raise PathNotAllowed(path) - self.repository = Repository(path, create) + self.repository = Repository(path, create, lock_wait=lock_wait) return self.repository.id @@ -122,7 +122,7 @@ class RemoteRepository: def __init__(self, name): self.name = name - def __init__(self, location, create=False): + def __init__(self, location, create=False, lock_wait=None): self.location = location self.preload_ids = [] self.msgid = 0 @@ -154,7 +154,7 @@ class RemoteRepository: raise ConnectionClosedWithHint('Is borg working on the server?') if version != 1: raise Exception('Server insisted on using unsupported protocol version %d' % version) - self.id = self.call('open', location.path, create) + self.id = self.call('open', location.path, create, lock_wait) def __del__(self): self.close() diff --git a/borg/repository.py b/borg/repository.py index 0127bca3..77d1d234 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -12,7 +12,7 @@ from zlib import crc32 from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify from .hashindex import NSIndex -from .locking import UpgradableLock +from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache MAX_OBJECT_SIZE = 20 * 1024 * 1024 @@ -51,7 +51,7 @@ class Repository: class ObjectNotFound(ErrorWithTraceback): """Object with key {} not found in repository {}.""" - def __init__(self, path, create=False, exclusive=False): + def __init__(self, path, create=False, exclusive=False, lock_wait=None): self.path = os.path.abspath(path) self.io = None self.lock = None @@ -59,7 +59,7 @@ class Repository: self._active_txn = False if create: self.create(self.path) - self.open(self.path, exclusive) + self.open(self.path, exclusive, lock_wait=lock_wait) def __del__(self): self.close() @@ -129,11 +129,11 @@ class Repository: self.replay_segments(replay_from, segments_transaction_id) return self.get_index_transaction_id() - def open(self, path, exclusive): + def open(self, path, exclusive, lock_wait=None): self.path = path if not os.path.isdir(path): raise self.DoesNotExist(path) - self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive).acquire() + self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive, timeout=lock_wait).acquire() self.config = ConfigParser(interpolation=None) self.config.read(os.path.join(self.path, 'config')) if 'repository' not in self.config.sections() or self.config.getint('repository', 'version') != 1: @@ -168,7 +168,7 @@ class Repository: self._active_txn = True try: self.lock.upgrade() - except UpgradableLock.ExclusiveLockFailed: + except (LockError, LockErrorT): # if upgrading the lock to exclusive fails, we do not have an # active transaction. this is important for "serve" mode, where # the repository instance lives on - even if exceptions happened. diff --git a/borg/testsuite/locking.py b/borg/testsuite/locking.py index 4b36e0ca..dc9d969c 100644 --- a/borg/testsuite/locking.py +++ b/borg/testsuite/locking.py @@ -2,7 +2,8 @@ import time import pytest -from ..locking import get_id, TimeoutTimer, ExclusiveLock , UpgradableLock, LockRoster, ADD, REMOVE, SHARED, EXCLUSIVE +from ..locking import get_id, TimeoutTimer, ExclusiveLock, UpgradableLock, LockRoster, \ + ADD, REMOVE, SHARED, EXCLUSIVE, LockTimeout ID1 = "foo", 1, 1 @@ -52,7 +53,7 @@ class TestExclusiveLock: def test_timeout(self, lockpath): with ExclusiveLock(lockpath, id=ID1): - with pytest.raises(ExclusiveLock.LockTimeout): + with pytest.raises(LockTimeout): ExclusiveLock(lockpath, id=ID2, timeout=0.1).acquire() @@ -92,6 +93,17 @@ class TestUpgradableLock: with UpgradableLock(lockpath, exclusive=True, id=ID2): pass + def test_timeout(self, lockpath): + with UpgradableLock(lockpath, exclusive=False, id=ID1): + with pytest.raises(LockTimeout): + UpgradableLock(lockpath, exclusive=True, id=ID2, timeout=0.1).acquire() + with UpgradableLock(lockpath, exclusive=True, id=ID1): + with pytest.raises(LockTimeout): + UpgradableLock(lockpath, exclusive=False, id=ID2, timeout=0.1).acquire() + with UpgradableLock(lockpath, exclusive=True, id=ID1): + with pytest.raises(LockTimeout): + UpgradableLock(lockpath, exclusive=True, id=ID2, timeout=0.1).acquire() + @pytest.fixture() def rosterpath(tmpdir): diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 2b99b83d..713f4402 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -6,7 +6,7 @@ from mock import patch from ..hashindex import NSIndex from ..helpers import Location, IntegrityError -from ..locking import UpgradableLock +from ..locking import UpgradableLock, LockFailed from ..remote import RemoteRepository, InvalidRPCMethod from ..repository import Repository from . import BaseTestCase @@ -158,9 +158,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): for name in os.listdir(self.repository.path): if name.startswith('index.'): os.unlink(os.path.join(self.repository.path, name)) - with patch.object(UpgradableLock, 'upgrade', side_effect=UpgradableLock.ExclusiveLockFailed) as upgrade: + with patch.object(UpgradableLock, 'upgrade', side_effect=LockFailed) as upgrade: self.reopen() - self.assert_raises(UpgradableLock.ExclusiveLockFailed, lambda: len(self.repository)) + self.assert_raises(LockFailed, lambda: len(self.repository)) upgrade.assert_called_once_with() def test_crash_before_write_index(self): From 1093894be83a5cd73d362b3d6b7f699f8c704553 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 16:53:33 +0100 Subject: [PATCH 097/321] UpgradableLock: release exclusive lock in case of exceptions also: add some comments about how to use the locks in the safest way --- borg/locking.py | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/borg/locking.py b/borg/locking.py index d3f309cb..cd54ca79 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -99,7 +99,14 @@ class NotMyLock(LockErrorT): class ExclusiveLock: - """An exclusive Lock based on mkdir fs operation being atomic""" + """An exclusive Lock based on mkdir fs operation being atomic. + + If possible, try to use the contextmanager here like: + with ExclusiveLock(...) as lock: + ... + This makes sure the lock is released again if the block is left, no + matter how (e.g. if an exception occurred). + """ def __init__(self, path, timeout=None, sleep=None, id=None): self.timeout = timeout self.sleep = sleep @@ -220,6 +227,12 @@ class UpgradableLock: Typically, write access to a resource needs an exclusive lock (1 writer, noone is allowed reading) and read access to a resource needs a shared lock (multiple readers are allowed). + + If possible, try to use the contextmanager here like: + with UpgradableLock(...) as lock: + ... + This makes sure the lock is released again if the block is left, no + matter how (e.g. if an exception occurred). """ def __init__(self, path, exclusive=False, sleep=None, timeout=None, id=None): self.path = path @@ -262,12 +275,18 @@ class UpgradableLock: timer = TimeoutTimer(self.timeout, sleep).start() while True: self._lock.acquire() - if remove is not None: - self._roster.modify(remove, REMOVE) - remove = None - if len(self._roster.get(SHARED)) == 0: - return # we are the only one and we keep the lock! - self._lock.release() + try: + if remove is not None: + self._roster.modify(remove, REMOVE) + remove = None + if len(self._roster.get(SHARED)) == 0: + return # we are the only one and we keep the lock! + except: + # avoid orphan lock when an exception happens here, e.g. Ctrl-C! + self._lock.release() + raise + else: + self._lock.release() if timer.timed_out_or_sleep(): raise LockTimeout(self.path) From 38994c78fc65db8d0c2f4f1899f98c1f31ae9184 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 20:50:53 +0100 Subject: [PATCH 098/321] implement borg break-lock REPO command, fixes #157 due to borg's architecture, breaking the repo lock needs first creating a repository object. this would usually try to get a lock and then block if there already is one. thus I added a flag to open without trying to create a lock. --- borg/archiver.py | 30 +++++++++++++++++++++++++++--- borg/cache.py | 5 +++++ borg/remote.py | 12 ++++++++---- borg/repository.py | 14 ++++++++++---- borg/testsuite/archiver.py | 4 ++++ 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 55563d3a..6e9a9422 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -42,11 +42,11 @@ class Archiver: self.verbose = verbose self.lock_wait = lock_wait - def open_repository(self, location, create=False, exclusive=False): + def open_repository(self, location, create=False, exclusive=False, lock=True): if location.proto == 'ssh': - repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait) + repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock) else: - repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait) + repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait, lock=lock) repository._location = location return repository @@ -573,6 +573,16 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_break_lock(self, args): + """Break the repository lock (e.g. in case it was left by a dead borg.""" + repository = self.open_repository(args.repository, lock=False) + try: + repository.break_lock() + Cache.break_lock(repository) + finally: + repository.close() + return self.exit_code + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -972,6 +982,20 @@ class Archiver: type=location_validator(archive=True), help='archive to display information about') + break_lock_epilog = textwrap.dedent(""" + This command breaks the repository and cache locks. + Please use carefully and only while no borg process (on any machine) is + trying to access the Cache or the Repository. + """) + subparser = subparsers.add_parser('break-lock', parents=[common_parser], + description=self.do_break_lock.__doc__, + epilog=break_lock_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_break_lock) + subparser.add_argument('repository', metavar='REPOSITORY', + type=location_validator(archive=False), + help='repository for which to break the locks') + prune_epilog = textwrap.dedent(""" The prune command prunes a repository by deleting archives not matching any of the specified retention options. This command is normally used by diff --git a/borg/cache.py b/borg/cache.py index 6116df49..c911283d 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -32,6 +32,11 @@ class Cache: class EncryptionMethodMismatch(Error): """Repository encryption method changed since last access, refusing to continue""" + @staticmethod + def break_lock(repository, path=None): + path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii')) + UpgradableLock(os.path.join(path, 'lock'), exclusive=True).break_lock() + def __init__(self, repository, key, manifest, path=None, sync=True, do_files=False, warn_if_unencrypted=True, lock_wait=None): self.lock = None diff --git a/borg/remote.py b/borg/remote.py index 1d5e9cab..3a6a5e5a 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -50,6 +50,7 @@ class RepositoryServer: # pragma: no cover 'rollback', 'save_key', 'load_key', + 'break_lock', ) def __init__(self, restrict_to_paths): @@ -97,7 +98,7 @@ class RepositoryServer: # pragma: no cover def negotiate(self, versions): return 1 - def open(self, path, create=False, lock_wait=None): + def open(self, path, create=False, lock_wait=None, lock=True): path = os.fsdecode(path) if path.startswith('/~'): path = path[1:] @@ -108,7 +109,7 @@ class RepositoryServer: # pragma: no cover break else: raise PathNotAllowed(path) - self.repository = Repository(path, create, lock_wait=lock_wait) + self.repository = Repository(path, create, lock_wait=lock_wait, lock=lock) return self.repository.id @@ -122,7 +123,7 @@ class RemoteRepository: def __init__(self, name): self.name = name - def __init__(self, location, create=False, lock_wait=None): + def __init__(self, location, create=False, lock_wait=None, lock=True): self.location = location self.preload_ids = [] self.msgid = 0 @@ -154,7 +155,7 @@ class RemoteRepository: raise ConnectionClosedWithHint('Is borg working on the server?') if version != 1: raise Exception('Server insisted on using unsupported protocol version %d' % version) - self.id = self.call('open', location.path, create, lock_wait) + self.id = self.call('open', location.path, create, lock_wait, lock) def __del__(self): self.close() @@ -308,6 +309,9 @@ class RemoteRepository: def load_key(self): return self.call('load_key') + def break_lock(self): + return self.call('break_lock') + def close(self): if self.p: self.p.stdin.close() diff --git a/borg/repository.py b/borg/repository.py index 77d1d234..f3189bc3 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -51,7 +51,7 @@ class Repository: class ObjectNotFound(ErrorWithTraceback): """Object with key {} not found in repository {}.""" - def __init__(self, path, create=False, exclusive=False, lock_wait=None): + def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True): self.path = os.path.abspath(path) self.io = None self.lock = None @@ -59,7 +59,7 @@ class Repository: self._active_txn = False if create: self.create(self.path) - self.open(self.path, exclusive, lock_wait=lock_wait) + self.open(self.path, exclusive, lock_wait=lock_wait, lock=lock) def __del__(self): self.close() @@ -129,11 +129,17 @@ class Repository: self.replay_segments(replay_from, segments_transaction_id) return self.get_index_transaction_id() - def open(self, path, exclusive, lock_wait=None): + def break_lock(self): + UpgradableLock(os.path.join(self.path, 'lock')).break_lock() + + def open(self, path, exclusive, lock_wait=None, lock=True): self.path = path if not os.path.isdir(path): raise self.DoesNotExist(path) - self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive, timeout=lock_wait).acquire() + if lock: + self.lock = UpgradableLock(os.path.join(path, 'lock'), exclusive, timeout=lock_wait).acquire() + else: + self.lock = None self.config = ConfigParser(interpolation=None) self.config.read(os.path.join(self.path, 'config')) if 'repository' not in self.config.sections() or self.config.getint('repository', 'version') != 1: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d61834d9..61471586 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -724,6 +724,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in('test-2', output) self.assert_not_in('something-else', output) + def test_break_lock(self): + self.cmd('init', self.repository_location) + self.cmd('break-lock', self.repository_location) + def test_usage(self): if self.FORK_DEFAULT: self.cmd(exit_code=0) From 7247043db0faa341d5bc7e9692dc6ddeb4d80674 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 22:08:30 +0100 Subject: [PATCH 099/321] get rid of C compiler warnings, fixes #391 --- borg/_chunker.c | 12 +++++------- borg/_hashindex.c | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/borg/_chunker.c b/borg/_chunker.c index 5f761c51..75821493 100644 --- a/borg/_chunker.c +++ b/borg/_chunker.c @@ -76,19 +76,18 @@ buzhash_update(uint32_t sum, unsigned char remove, unsigned char add, size_t len } typedef struct { - int window_size, chunk_mask, min_size; - size_t buf_size; + uint32_t chunk_mask; uint32_t *table; uint8_t *data; PyObject *fd; int fh; int done, eof; - size_t remaining, position, last; + size_t min_size, buf_size, window_size, remaining, position, last; off_t bytes_read, bytes_yielded; } Chunker; static Chunker * -chunker_init(int window_size, int chunk_mask, int min_size, int max_size, uint32_t seed) +chunker_init(size_t window_size, uint32_t chunk_mask, size_t min_size, size_t max_size, uint32_t seed) { Chunker *c = calloc(sizeof(Chunker), 1); c->window_size = window_size; @@ -203,9 +202,8 @@ PyBuffer_FromMemory(void *data, Py_ssize_t len) static PyObject * chunker_process(Chunker *c) { - uint32_t sum, chunk_mask = c->chunk_mask, min_size = c->min_size, window_size = c->window_size; - int n = 0; - int old_last; + uint32_t sum, chunk_mask = c->chunk_mask; + size_t n = 0, old_last, min_size = c->min_size, window_size = c->window_size; if(c->done) { if(c->bytes_read == c->bytes_yielded) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index e1ff936f..c551aec1 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -171,7 +171,7 @@ hashindex_read(const char *path) goto fail; } buckets_length = (off_t)_le32toh(header.num_buckets) * (header.key_size + header.value_size); - if(length != sizeof(HashHeader) + buckets_length) { + if((size_t) length != sizeof(HashHeader) + buckets_length) { EPRINTF_MSG_PATH(path, "Incorrect file length (expected %ju, got %ju)", (uintmax_t) sizeof(HashHeader) + buckets_length, (uintmax_t) length); goto fail; @@ -275,7 +275,7 @@ hashindex_write(HashIndex *index, const char *path) EPRINTF_PATH(path, "fwrite header failed"); ret = 0; } - if(fwrite(index->buckets, 1, buckets_length, fd) != buckets_length) { + if(fwrite(index->buckets, 1, buckets_length, fd) != (size_t) buckets_length) { EPRINTF_PATH(path, "fwrite buckets failed"); ret = 0; } From adb35ab07f86aeac02f4e37bad41f5fa27f07842 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 21 Nov 2015 22:51:59 +0100 Subject: [PATCH 100/321] include system info below traceback, fixes #324 --- borg/archiver.py | 10 +++++----- borg/helpers.py | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index b65a7630..e45ace68 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -20,7 +20,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ - dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, \ + dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .logger import create_logger, setup_logging logger = create_logger() @@ -1224,16 +1224,16 @@ def main(): # pragma: no cover except Error as e: msg = e.get_message() if e.traceback: - msg += "\n%s" % traceback.format_exc() + msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) exit_code = e.exit_code except RemoteRepository.RPCError as e: - msg = 'Remote Exception.\n%s' % str(e) + msg = 'Remote Exception.\n%s\n%s' % (str(e), sysinfo()) exit_code = EXIT_ERROR except Exception: - msg = 'Local Exception.\n%s' % traceback.format_exc() + msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo()) exit_code = EXIT_ERROR except KeyboardInterrupt: - msg = 'Keyboard interrupt.\n%s' % traceback.format_exc() + msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo()) exit_code = EXIT_ERROR if msg: logger.error(msg) diff --git a/borg/helpers.py b/borg/helpers.py index b4fbb7e5..343bcc07 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -14,6 +14,7 @@ except ImportError: TerminalSize = namedtuple('TerminalSize', ['columns', 'lines']) return TerminalSize(int(os.environ.get('COLUMNS', fallback[0])), int(os.environ.get('LINES', fallback[1]))) import sys +import platform import time import unicodedata @@ -887,3 +888,13 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, if retry_msg: print(retry_msg, file=ofile, end='') ofile.flush() + + +def sysinfo(): + info = [] + info.append('Platform: %s' % (' '.join(platform.uname()), )) + if sys.platform.startswith('linux'): + info.append('Linux: %s %s %s LibC: %s %s' % (platform.linux_distribution() + platform.libc_ver())) + info.append('Python: %s %s' % (platform.python_implementation(), platform.python_version())) + info.append('') + return '\n'.join(info) From 2b8b31bca59033afba304dc8090ec6e34a601b53 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Nov 2015 15:39:31 +0100 Subject: [PATCH 101/321] update pytest-benchmark requirement it's released now. \o/ --- requirements.d/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 28195f54..dd9c3683 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -2,5 +2,5 @@ tox mock pytest pytest-cov<2.0.0 -pytest-benchmark==3.0.0rc1 +pytest-benchmark>=3.0.0 Cython From 0196d80b28e194c5d331a0cbc160dd27746523df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 22 Nov 2015 21:24:37 -0500 Subject: [PATCH 102/321] fix progress tests on travis we now check if we really have a terminal before doing the fancy auto-detection testing --- borg/testsuite/archiver.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 48b9da0c..928b12be 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -670,6 +670,19 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(len(manifest.archives), 0) def test_progress(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', self.repository_location) + # progress forced on + output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input') + self.assert_in("\r", output) + # progress forced off + output = self.cmd('create', '--no-progress', self.repository_location + '::test5', 'input') + self.assert_not_in("\r", output) + + @unittest.skipUnless(sys.stdout.isatty(), 'need a tty to test auto-detection') + def test_progress_tty(self): + """test that the --progress and --no-progress flags work, + overriding defaults from the terminal auto-detection""" self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) # without a terminal, no progress expected From 2ac515a5f7696a7e4dc6f2691766c4b7e1b1ea1c Mon Sep 17 00:00:00 2001 From: anarcat Date: Mon, 23 Nov 2015 12:41:20 -0500 Subject: [PATCH 103/321] fix typos --- borg/testsuite/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 928b12be..2d31a830 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -697,10 +697,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): # with a terminal, progress forced on output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input', fork=True) self.assert_in("\r", output) - # without a termainl, progress forced off + # without a terminal, progress forced off output = self.cmd('create', '--no-progress', self.repository_location + '::test5', 'input', fork=False) self.assert_not_in("\r", output) - # with a termainl, progress forced off + # with a terminal, progress forced off output = self.cmd('create', '--no-progress', self.repository_location + '::test6', 'input', fork=True) self.assert_not_in("\r", output) From a75d77226b3ab567bb14640f6143a26a8764cad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 19:37:43 -0500 Subject: [PATCH 104/321] add test for the weird unchanged file status this tests the behaviour found in #403 and documented in #418, but doesn't fail on the unexpected A --- borg/testsuite/archiver.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8bb1a58d..9d78988c 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -704,6 +704,25 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', '--no-progress', self.repository_location + '::test6', 'input', fork=True) self.assert_not_in("\r", output) + def test_file_status(self): + """test that various file status show expected results + + clearly incomplete: only tests for the weird "unchanged" status for now""" + now = time.time() + self.create_regular_file('file1', size=1024 * 80) + os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago + self.create_regular_file('file2', size=1024 * 80) + self.cmd('init', self.repository_location) + output = self.cmd('create', '--verbose', self.repository_location + '::test', 'input') + self.assert_in("A input/file1", output) + self.assert_in("A input/file2", output) + # should find first file as unmodified + output = self.cmd('create', '--verbose', self.repository_location + '::test1', 'input') + self.assert_in("U input/file1", output) + # this is expected, although surprising, for why, see: + # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file + self.assert_in("A input/file2", output) + def test_cmdline_compatibility(self): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) From af7b17960ec0cea0196b89c9c82d31c3c5dbbae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 19:43:41 -0500 Subject: [PATCH 105/321] clarify documentation on the A status oddity --- docs/faq.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 5487532b..8c88125c 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -168,9 +168,9 @@ repo. It will then be able to check using CRCs and HMACs. I am seeing 'A' (added) status for a unchanged file!? ----------------------------------------------------- -The files cache (which is used to determine whether |project_name| already -"knows" / has backed up a file and if so, to skip the file from chunking) -does intentionally *not* contain files that: +The files cache is used to determine whether |project_name| already +"knows" / has backed up a file and if so, to skip the file from +chunking. It does intentionally *not* contain files that: - have >= 10 as "entry age" (|project_name| has not seen this file for a while) - have a modification time (mtime) same as the newest mtime in the created @@ -191,6 +191,10 @@ will often be the same and already stored in the repo (except in the above mentioned rare condition), it will just re-use them as usual and not store new data chunks. +Since only the files cache is used in the display of files status, +those files are reported as being added when, really, chunks are +already used. + Why was Borg forked from Attic? ------------------------------- From 48bb4c326db79c3cd8acd723575993e19294631f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 19:48:33 -0500 Subject: [PATCH 106/321] cross-reference the status oddity in the usage --- docs/faq.rst | 2 ++ docs/usage.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 8c88125c..ef6ca661 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -165,6 +165,8 @@ Yes, if you want to detect accidental data damage (like bit rot), use the If you want to be able to detect malicious tampering also, use a encrypted repo. It will then be able to check using CRCs and HMACs. +.. _a_status_oddity: + I am seeing 'A' (added) status for a unchanged file!? ----------------------------------------------------- diff --git a/docs/usage.rst b/docs/usage.rst index 5bba8390..1db3bc59 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -444,7 +444,7 @@ A uppercase character represents the status of a regular file relative to the is not used). Metadata is stored in any case and for 'A' and 'M' also new data chunks are stored. For 'U' all data chunks refer to already existing chunks. -- 'A' = regular file, added +- 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) - 'M' = regular file, modified - 'U' = regular file, unchanged - 'E' = regular file, an error happened while accessing/reading *this* file From a8227aeda09d8778214446b14c368f8f1b335c25 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 24 Nov 2015 17:38:12 +0100 Subject: [PATCH 107/321] update CHANGES --- docs/changes.rst | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index f60edd59..d87dba7d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,49 @@ Changelog ========= +Version 0.29.0 +-------------- + +Compatibility notes: + +- when upgrading to 0.29.0 you need to upgrade client as well as server + installations due to the locking related changes. +- the default waiting time for a lock changed from infinity to 1 second for a + better interactive user experience. if the repo you want to access is + currently locked, borg will now terminate after 1s with an error message. + if you have scripts that shall wait for the lock for a longer time, use + --lock-wait N (with N being the maximum wait time in seconds). + +Bug fixes: + +- avoid creation of an orphan lock for one case, see #285 +- --keep-tag-files: fix file mode and multiple tag files in one directory, #432 + +New features: + +- implement --lock-wait, support timeout for UpgradableLock, fixes #210 +- implement borg break-lock command, fixes #157 +- include system info below traceback, fixes #324 +- use ISO-8601 date and time format, fixes #375 +- add --log-level to set the level of the builtin logging configuration, fixes #426 +- configure logging via env var BORG_LOGGING_CONF +- add a --no-progress flag to forcibly disable progress info + +Other changes: + +- fix progress tests on travis +- get rid of C compiler warnings, fixes #391 +- upgrade OS X FUSE to 3.0.9 on the OS X binary build system +- docs: + + - document new mailing list borgbackup@python.org + - readthedocs: color and logo improvements + - more precise binary installation steps + - update release procedure docs about OS X FUSE + - FAQ entry about unexpected 'A' status for unchanged file(s), fixes #403 + - add docs about 'E' file status + + Version 0.28.2 -------------- From 6b265f2a53a07be5796d72c525baacef0424482e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 12:00:57 -0500 Subject: [PATCH 108/321] alias --verbose to --log-level=info print_verbose is now simply logger.info() and is always displayed if log level allows it. this affects only the `prune` and `mount` commands which were the only users of the --verbose option. the additional display is which archives are kept and pruned and a single message when the fileystem is mounted. files iteration in create and extract is now printed through a separate function which will be later controled through a topical flag. --- borg/archiver.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 1c066d95..e7c9cb57 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -51,9 +51,8 @@ class ToggleAction(argparse.Action): class Archiver: - def __init__(self, verbose=False, lock_wait=None): + def __init__(self, lock_wait=None): self.exit_code = EXIT_SUCCESS - self.verbose = verbose self.lock_wait = lock_wait def open_repository(self, location, create=False, exclusive=False, lock=True): @@ -74,10 +73,8 @@ class Archiver: self.exit_code = EXIT_WARNING # we do not terminate here, so it is a warning logger.warning(msg) - def print_verbose(self, msg, *args): - if self.verbose: - msg = args and msg % args or msg - logger.info(msg) + def print_file_status(self, status, path): + logger.info("1s %s", status, remove_surrogates(path)) def do_serve(self, args): """Start in server mode. This command is usually not used manually. @@ -166,7 +163,7 @@ class Archiver: self.print_warning('%s: %s', path, e) else: status = '-' - self.print_verbose("%1s %s", status, remove_surrogates(path)) + self.print_file_status(status, path) continue path = os.path.normpath(path) if args.one_file_system: @@ -265,9 +262,7 @@ class Archiver: status = '?' # need to add a status code somewhere else: status = '-' # dry run, item was not backed up - # output ALL the stuff - it can be easily filtered using grep. - # even stuff considered unchanged might be interesting. - self.print_verbose("%1s %s", status, remove_surrogates(path)) + self.print_file_status(status, path) def do_extract(self, args): """Extract archive contents""" @@ -295,7 +290,7 @@ class Archiver: if not args.dry_run: while dirs and not item[b'path'].startswith(dirs[-1][b'path']): archive.extract_item(dirs.pop(-1), stdout=stdout) - self.print_verbose(remove_surrogates(orig_path)) + logger.info(remove_surrogates(orig_path)) try: if dry_run: archive.extract_item(item, dry_run=True) @@ -381,7 +376,7 @@ class Archiver: else: archive = None operations = FuseOperations(key, repository, manifest, archive) - self.print_verbose("Mounting filesystem") + logger.info("Mounting filesystem") try: operations.mount(args.mountpoint, args.options, args.foreground) except RuntimeError: @@ -484,12 +479,12 @@ class Archiver: to_delete = [a for a in archives if a not in keep] stats = Statistics() for archive in keep: - self.print_verbose('Keeping archive: %s' % format_archive(archive)) + logger.info('Keeping archive: %s' % format_archive(archive)) for archive in to_delete: if args.dry_run: - self.print_verbose('Would prune: %s' % format_archive(archive)) + logger.info('Would prune: %s' % format_archive(archive)) else: - self.print_verbose('Pruning archive: %s' % format_archive(archive)) + logger.info('Pruning archive: %s' % format_archive(archive)) Archive(repository, key, manifest, archive.name, cache).delete(stats) if to_delete and not args.dry_run: manifest.write() @@ -666,8 +661,9 @@ class Archiver: def build_parser(self, args=None, prog=None): common_parser = argparse.ArgumentParser(add_help=False, prog=prog) - common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, - help='verbose output') + common_parser.add_argument('-v', '--verbose', dest='log_level', + action='store_const', const='info', default='info', + help='verbose output, same as --log-level=info') common_parser.add_argument('--log-level', dest='log_level', default='info', metavar='LEVEL', choices=('debug', 'info', 'warning', 'error', 'critical'), help='set the log level to LEVEL, default: %(default)s)') @@ -1180,7 +1176,6 @@ class Archiver: def run(self, args): os.umask(args.umask) # early, before opening files - self.verbose = args.verbose self.lock_wait = args.lock_wait RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask From 9899eaf241b1e6351733bf1d2db68046edec5994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 12:10:21 -0500 Subject: [PATCH 109/321] silence file listing unless --changed is present --- borg/archiver.py | 6 +++++- borg/testsuite/archiver.py | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index e7c9cb57..15c6caf8 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -74,7 +74,8 @@ class Archiver: logger.warning(msg) def print_file_status(self, status, path): - logger.info("1s %s", status, remove_surrogates(path)) + if self.args.changed: + logger.info("%1s %s" % (status, remove_surrogates(path))) def do_serve(self, args): """Start in server mode. This command is usually not used manually. @@ -801,6 +802,8 @@ class Archiver: help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: %(default)s""") + subparser.add_argument('--changed', action='store_true', dest='changed', default=False, + help="""display which files were added to the archive""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') @@ -1171,6 +1174,7 @@ class Archiver: args = self.preprocess_args(args) parser = self.build_parser(args) args = parser.parse_args(args or ['-h']) + self.args = args update_excludes(args) return args diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 9d78988c..a066f6ec 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -723,6 +723,27 @@ class ArchiverTestCase(ArchiverTestCaseBase): # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file self.assert_in("A input/file2", output) + def test_create_topical(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', self.repository_location) + # no listing by default + output = self.cmd('create', self.repository_location + '::test', 'input') + self.assert_not_in('file1', output) + # shouldn't be listed even if unchanged + output = self.cmd('create', self.repository_location + '::test0', 'input') + self.assert_not_in('file1', output) + # should list the file as unchanged + #output = self.cmd('create', '--unchanged', self.repository_location + '::test1', 'input') + #self.assert_in('file1', output) + # should *not* list the file as changed + #output = self.cmd('create', '--changed', self.repository_location + '::test2', 'input') + #self.assert_not_in('file1', output) + # change the file + self.create_regular_file('file1', size=1024 * 100) + # should list the file as changed + output = self.cmd('create', '--changed', self.repository_location + '::test3', 'input') + self.assert_in('file1', output) + def test_cmdline_compatibility(self): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) From 8d3d1c22d68b21b09553417fce08b938e9404ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 12:38:11 -0500 Subject: [PATCH 110/321] silence borg by default this also prints file status on stderr directly, bypassing the logger as we do with other topical flags (like progress and status) --- borg/archiver.py | 6 +++--- borg/testsuite/archiver.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 15c6caf8..6c2875d1 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -75,7 +75,7 @@ class Archiver: def print_file_status(self, status, path): if self.args.changed: - logger.info("%1s %s" % (status, remove_surrogates(path))) + print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) def do_serve(self, args): """Start in server mode. This command is usually not used manually. @@ -663,9 +663,9 @@ class Archiver: def build_parser(self, args=None, prog=None): common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser.add_argument('-v', '--verbose', dest='log_level', - action='store_const', const='info', default='info', + action='store_const', const='info', default='warning', help='verbose output, same as --log-level=info') - common_parser.add_argument('--log-level', dest='log_level', default='info', metavar='LEVEL', + common_parser.add_argument('--log-level', dest='log_level', default='warning', metavar='LEVEL', choices=('debug', 'info', 'warning', 'error', 'critical'), help='set the log level to LEVEL, default: %(default)s)') common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index a066f6ec..c458f97f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -947,13 +947,13 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): return archive, repository def test_check_usage(self): - output = self.cmd('check', self.repository_location, exit_code=0) + output = self.cmd('check', '--log-level=info', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) - output = self.cmd('check', '--repository-only', self.repository_location, exit_code=0) + output = self.cmd('check', '--log-level=info', '--repository-only', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) self.assert_not_in('Starting archive consistency check', output) - output = self.cmd('check', '--archives-only', self.repository_location, exit_code=0) + output = self.cmd('check', '--log-level=info', '--archives-only', self.repository_location, exit_code=0) self.assert_not_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) From a062e8f821af35075f4d4a4c0372e8560f792775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 12:52:24 -0500 Subject: [PATCH 111/321] update documentation to follow changes --- docs/usage.rst | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 1db3bc59..57702452 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -21,12 +21,6 @@ Supported levels: ``debug``, ``info``, ``warning``, ``error``, ``critical``. All log messages created with at least the given level will be output. -Amount of informational output -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -For some subcommands, using the ``-v`` or ``--verbose`` option will give you -more informational output (at ``info`` level). - Return codes ~~~~~~~~~~~~ @@ -435,7 +429,7 @@ Here are misc. notes about topics that are maybe not covered in enough detail in Item flags ~~~~~~~~~~ -`borg create -v` outputs a verbose list of all files, directories and other +`borg create --changed` outputs a verbose list of all files, directories and other file system items it considered. For each item, it prefixes a single-letter flag that indicates type and/or status of the item. From fce5aed5bbab6c23299cb30da2d40aa515155245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 17:30:49 -0500 Subject: [PATCH 112/321] move changed with other topical flags we need to have a sane default there otherwise the option may not be defined in some sub-commands and will crash --- borg/archiver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6c2875d1..1c396a92 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -74,7 +74,7 @@ class Archiver: logger.warning(msg) def print_file_status(self, status, path): - if self.args.changed: + if self.changed: print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) def do_serve(self, args): @@ -1174,13 +1174,14 @@ class Archiver: args = self.preprocess_args(args) parser = self.build_parser(args) args = parser.parse_args(args or ['-h']) - self.args = args update_excludes(args) return args def run(self, args): os.umask(args.umask) # early, before opening files self.lock_wait = args.lock_wait + self.changed = getattr(args, 'changed', False) + self.unchanged = getattr(args, 'unchanged', False) RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask setup_logging(level=args.log_level) # do not use loggers before this! From 1785ca54ba82ba851905478446d9bf00a3f9f597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 23 Nov 2015 19:54:30 -0500 Subject: [PATCH 113/321] do not display unchanged files by default add a --unchanged topical file to display those files --- borg/archiver.py | 10 ++++++++-- borg/testsuite/archiver.py | 11 +++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 1c396a92..eefe8136 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -74,8 +74,12 @@ class Archiver: logger.warning(msg) def print_file_status(self, status, path): - if self.changed: - print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) + if status == 'U': + if self.unchanged: + print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) + else: + if self.changed: + print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) def do_serve(self, args): """Start in server mode. This command is usually not used manually. @@ -804,6 +808,8 @@ class Archiver: and the path being processed, default: %(default)s""") subparser.add_argument('--changed', action='store_true', dest='changed', default=False, help="""display which files were added to the archive""") + subparser.add_argument('--unchanged', action='store_true', dest='unchanged', default=False, + help="""display which files were *not* added to the archive""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index c458f97f..fcabb35d 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -724,7 +724,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in("A input/file2", output) def test_create_topical(self): + now = time.time() self.create_regular_file('file1', size=1024 * 80) + os.utime('input/file1', (now-5, now-5)) + self.create_regular_file('file2', size=1024 * 80) self.cmd('init', self.repository_location) # no listing by default output = self.cmd('create', self.repository_location + '::test', 'input') @@ -733,11 +736,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', self.repository_location + '::test0', 'input') self.assert_not_in('file1', output) # should list the file as unchanged - #output = self.cmd('create', '--unchanged', self.repository_location + '::test1', 'input') - #self.assert_in('file1', output) + output = self.cmd('create', '--unchanged', self.repository_location + '::test1', 'input') + self.assert_in('file1', output) # should *not* list the file as changed - #output = self.cmd('create', '--changed', self.repository_location + '::test2', 'input') - #self.assert_not_in('file1', output) + output = self.cmd('create', '--changed', self.repository_location + '::test2', 'input') + self.assert_not_in('file1', output) # change the file self.create_regular_file('file1', size=1024 * 100) # should list the file as changed From b09643e14f911e0f14a9109560a0177a8eaa7444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Tue, 24 Nov 2015 12:11:43 -0500 Subject: [PATCH 114/321] change file status test and cleanup last ref to --verbose this ports the changes here to #445 --- borg/testsuite/archiver.py | 4 ++-- docs/usage.rst | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index fcabb35d..8542c338 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -713,11 +713,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago self.create_regular_file('file2', size=1024 * 80) self.cmd('init', self.repository_location) - output = self.cmd('create', '--verbose', self.repository_location + '::test', 'input') + output = self.cmd('create', '--changed', '--unchanged', self.repository_location + '::test', 'input') self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) # should find first file as unmodified - output = self.cmd('create', '--verbose', self.repository_location + '::test1', 'input') + output = self.cmd('create', '--changed', '--unchanged', self.repository_location + '::test1', 'input') self.assert_in("U input/file1", output) # this is expected, although surprising, for why, see: # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file diff --git a/docs/usage.rst b/docs/usage.rst index 57702452..67e6a039 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -430,7 +430,8 @@ Item flags ~~~~~~~~~~ `borg create --changed` outputs a verbose list of all files, directories and other -file system items it considered. For each item, it prefixes a single-letter +file system items it considered, with the exception of unchanged files +(for this, also add `--unchanged`). For each item, it prefixes a single-letter flag that indicates type and/or status of the item. A uppercase character represents the status of a regular file relative to the @@ -440,7 +441,7 @@ chunks are stored. For 'U' all data chunks refer to already existing chunks. - 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) - 'M' = regular file, modified -- 'U' = regular file, unchanged +- 'U' = regular file, unchanged (only if `--unchanged` is specified) - 'E' = regular file, an error happened while accessing/reading *this* file A lowercase character means a file type other than a regular file, From 610300c1cec8f21dee7bcbba386d81032cdedbac Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 1 Dec 2015 21:18:58 +0100 Subject: [PATCH 115/321] misc. hash table tuning BUCKET_UPPER_LIMIT: 90% load degrades hash table performance severely, so I lowered that to 75% (which is a usual value - java uses 75%, python uses 66%). I chose the higher value of both because we also should not consume too much memory, considering the RAM usage already is rather high. MIN_BUCKETS: I can't explain why, but benchmarks showed that choosing 2^N as table size severely degrades performance (by 3 orders of magnitude!). So a prime start value improves this a lot, even if we later still use the grow-by-2x algorithm. hashindex_resize: removed the hashindex_get() call as we already know that the values come at key + key_size address. hashindex_init: do not calloc X*Y elements of size 1, but rather X elements of size Y. Makes the code simpler, not sure if it affects performance. The tests needed fixing as the resulting hashtable blob is now of course different due to the above changes, so its sha hash changed. --- borg/_hashindex.c | 13 ++++++------- borg/testsuite/hashindex.py | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index c551aec1..a61c644b 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -44,8 +44,8 @@ typedef struct { #define DELETED _htole32(0xfffffffe) #define MAX_BUCKET_SIZE 512 #define BUCKET_LOWER_LIMIT .25 -#define BUCKET_UPPER_LIMIT .90 -#define MIN_BUCKETS 1024 +#define BUCKET_UPPER_LIMIT .75 /* don't go higher than 0.75, otherwise performance severely suffers! */ +#define MIN_BUCKETS 1031 /* must be prime, otherwise performance breaks down! */ #define MAX(x, y) ((x) > (y) ? (x): (y)) #define BUCKET_ADDR(index, idx) (index->buckets + (idx * index->bucket_size)) @@ -113,12 +113,13 @@ hashindex_resize(HashIndex *index, int capacity) { HashIndex *new; void *key = NULL; + int32_t key_size = index->key_size; - if(!(new = hashindex_init(capacity, index->key_size, index->value_size))) { + if(!(new = hashindex_init(capacity, key_size, index->value_size))) { return 0; } while((key = hashindex_next_key(index, key))) { - hashindex_set(new, key, hashindex_get(index, key)); + hashindex_set(new, key, key + key_size); } free(index->buckets); index->buckets = new->buckets; @@ -218,7 +219,6 @@ fail: static HashIndex * hashindex_init(int capacity, int key_size, int value_size) { - off_t buckets_length; HashIndex *index; int i; capacity = MAX(MIN_BUCKETS, capacity); @@ -227,8 +227,7 @@ hashindex_init(int capacity, int key_size, int value_size) EPRINTF("malloc header failed"); return NULL; } - buckets_length = (off_t)capacity * (key_size + value_size); - if(!(index->buckets = calloc(buckets_length, 1))) { + if(!(index->buckets = calloc(capacity, key_size + value_size))) { EPRINTF("malloc buckets failed"); free(index); return NULL; diff --git a/borg/testsuite/hashindex.py b/borg/testsuite/hashindex.py index bbefeb05..0421ed8c 100644 --- a/borg/testsuite/hashindex.py +++ b/borg/testsuite/hashindex.py @@ -51,11 +51,11 @@ class HashIndexTestCase(BaseTestCase): def test_nsindex(self): self._generic_test(NSIndex, lambda x: (x, x), - '861d6d60069ea45e39d36bed2bdc1d0c07981e0641955f897ac6848be429abac') + '80fba5b40f8cf12f1486f1ba33c9d852fb2b41a5b5961d3b9d1228cf2aa9c4c9') def test_chunkindex(self): self._generic_test(ChunkIndex, lambda x: (x, x, x), - '69464bd0ebbc5866b9f95d838bc48617d21bfe3dcf294682a5c21a2ef6b9dc0b') + '1d71865e72e3c3af18d3c7216b6fa7b014695eaa3ed7f14cf9cd02fba75d1c95') def test_resize(self): n = 2000 # Must be >= MIN_BUCKETS From 5280c0830e257e90959578c64e07041da2768477 Mon Sep 17 00:00:00 2001 From: Vlad Date: Wed, 2 Dec 2015 00:16:25 +0200 Subject: [PATCH 116/321] Borg moved to Arch Linux [community] --- docs/installation.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index e83f9e7f..271fd086 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -32,13 +32,13 @@ yet. ============ ===================== ======= Distribution Source Command ============ ===================== ======= -Arch Linux AUR_ +Arch Linux `[community]`_ ``pacman -Syu borg`` Debian `unstable/sid`_ ``apt install borgbackup`` Ubuntu `Xenial Xerus 16.04`_ ``apt install borgbackup`` OS X `Brew cask`_ ``brew cask install borgbackup`` ============ ===================== ======= -.. _AUR: https://aur.archlinux.org/packages/borgbackup/ +.. _[community]: https://www.archlinux.org/packages/?name=borg .. _unstable/sid: https://packages.debian.org/sid/borgbackup .. _Xenial Xerus 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup .. _Brew cask: http://caskroom.io/ From e6d3720d9b731fca6a347dd522ba22879bc14639 Mon Sep 17 00:00:00 2001 From: Stavros Korokithakis Date: Wed, 2 Dec 2015 01:48:08 +0200 Subject: [PATCH 117/321] Clarify encryption. --- docs/quickstart.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 6890ab94..b2f30aa0 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -152,16 +152,17 @@ Repository encryption is enabled at repository creation time:: When repository encryption is enabled all data is encrypted using 256-bit AES_ encryption and the integrity and authenticity is verified using `HMAC-SHA256`_. -All data is encrypted before being written to the repository. This means that -an attacker who manages to compromise the host containing an encrypted -archive will not be able to access any of the data. +All data is encrypted on the client before being written to the repository. This +means that an attacker who manages to compromise the host containing an +encrypted archive will not be able to access any of the data, even as the backup +is being made. |project_name| supports different methods to store the AES and HMAC keys. ``repokey`` mode The key is stored inside the repository (in its "config" file). Use this mode if you trust in your good passphrase giving you enough - protection. + protection. The repository server never sees the plaintext key. ``keyfile`` mode The key is stored on your local disk (in ``~/.borg/keys/``). From 7a1316cb793f3de08e516c20492165bfaa3a5fae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Nov 2015 19:53:44 +0100 Subject: [PATCH 118/321] implement ProgressIndicators, use it for repo check and segment replay, fixes #195, fixes #188 --- borg/archiver.py | 5 +-- borg/helpers.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++ borg/repository.py | 23 ++++++++++++-- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index e88dc51f..9e2d2eb5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -93,10 +93,7 @@ class Archiver: env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): return EXIT_ERROR if not args.archives_only: - logger.info('Starting repository check...') - if repository.check(repair=args.repair): - logger.info('Repository check complete, no problems found.') - else: + if not repository.check(repair=args.repair): return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( repository, repair=args.repair, archive=args.repository.archive, last=args.last): diff --git a/borg/helpers.py b/borg/helpers.py index 343bcc07..8516496c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -890,6 +890,82 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, ofile.flush() +class ProgressIndicatorPercent: + def __init__(self, total, step=5, start=0, same_line=False, msg="%3.0f%%", file=sys.stderr): + """ + Percentage-based progress indicator + + :param total: total amount of items + :param step: step size in percent + :param start: at which percent value to start + :param file: output file, default: sys.stderr + """ + self.counter = 0 # 0 .. (total-1) + self.total = total + self.trigger_at = start # output next percentage value when reaching (at least) this + self.step = step + self.file = file + self.msg = msg + self.same_line = same_line + + def progress(self, current=None): + if current is None: + current = self.counter + else: + self.counter = current + self.counter += 1 + pct = current * 100 / self.total + if pct >= self.trigger_at: + self.trigger_at += self.step + return pct + + def show(self, current=None): + pct = self.progress(current) + if pct is not None: + return self.output(pct) + + def output(self, percent): + print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n') + + def finish(self): + if self.same_line: + print(" " * len(self.msg % 100.0), file=self.file, end='\r') + + + +class ProgressIndicatorEndless: + def __init__(self, step=10, file=sys.stderr): + """ + Progress indicator (long row of dots) + + :param step: every Nth call, call the func + :param file: output file, default: sys.stderr + """ + self.counter = 0 # call counter + self.triggered = 0 # increases 1 per trigger event + self.step = step # trigger every calls + self.file = file + + def progress(self): + self.counter += 1 + trigger = self.counter % self.step == 0 + if trigger: + self.triggered += 1 + return trigger + + def show(self): + trigger = self.progress() + if trigger: + return self.output(self.triggered) + + def output(self, triggered): + print('.', end='', file=self.file) # python 3.3 gives us flush=True + self.file.flush() + + def finish(self): + print(file=self.file) + + def sysinfo(): info = [] info.append('Platform: %s' % (' '.join(platform.uname()), )) diff --git a/borg/repository.py b/borg/repository.py index f3189bc3..49e3ac23 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -10,7 +10,8 @@ import shutil import struct from zlib import crc32 -from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify +from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, \ + unhexlify, ProgressIndicatorPercent from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache @@ -243,13 +244,17 @@ class Repository: def replay_segments(self, index_transaction_id, segments_transaction_id): self.prepare_txn(index_transaction_id, do_cleanup=False) try: - for segment, filename in self.io.segment_iterator(): + segment_count = sum(1 for _ in self.io.segment_iterator()) + pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%", same_line=True) + for i, (segment, filename) in enumerate(self.io.segment_iterator()): + pi.show(i) if index_transaction_id is not None and segment <= index_transaction_id: continue if segment > segments_transaction_id: break objects = self.io.iter_objects(segment) self._update_index(segment, objects) + pi.finish() self.write_index() finally: self.rollback() @@ -299,6 +304,7 @@ class Repository: error_found = True logger.error(msg) + logger.info('Starting repository check') assert not self._active_txn try: transaction_id = self.get_transaction_id() @@ -314,7 +320,10 @@ class Repository: self.io.cleanup(transaction_id) segments_transaction_id = self.io.get_segments_transaction_id() self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! - for segment, filename in self.io.segment_iterator(): + segment_count = sum(1 for _ in self.io.segment_iterator()) + pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.0f%%", same_line=True) + for i, (segment, filename) in enumerate(self.io.segment_iterator()): + pi.show(i) if segment > transaction_id: continue try: @@ -326,6 +335,7 @@ class Repository: self.io.recover_segment(segment, filename) objects = list(self.io.iter_objects(segment)) self._update_index(segment, objects, report_error) + pi.finish() # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found if repair and segments_transaction_id is None: @@ -345,6 +355,13 @@ class Repository: self.compact_segments() self.write_index() self.rollback() + if error_found: + if repair: + logger.info('Completed repository check, errors found and repaired.') + else: + logger.info('Completed repository check, errors found.') + else: + logger.info('Completed repository check, no problems found.') return not error_found or repair def rollback(self): From fe6916bd22ba130079a38b8ad2e12257f6fcac56 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 2 Dec 2015 01:26:26 +0100 Subject: [PATCH 119/321] refactor upgrade progress indication code to use ProgressIndicatorPercent --- borg/upgrader.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index 2a8a977d..81981498 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -7,7 +7,7 @@ import shutil import sys import time -from .helpers import get_keys_dir, get_cache_dir +from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent from .locking import UpgradableLock from .repository import Repository, MAGIC from .key import KeyfileKey, KeyfileNotFoundError @@ -65,17 +65,15 @@ class AtticRepositoryUpgrader(Repository): luckily the magic string length didn't change so we can just replace the 8 first bytes of all regular files in there.""" logger.info("converting %d segments..." % len(segments)) - i = 0 - for filename in segments: - i += 1 - print("\rconverting segment %d/%d, %.2f%% done (%s)" - % (i, len(segments), 100*float(i)/len(segments), filename), - end='', file=sys.stderr) + segment_count = len(segments) + pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", same_line=True) + for i, filename in enumerate(segments): + pi.show(i) if dryrun: time.sleep(0.001) else: AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace) - print(file=sys.stderr) + pi.finish() @staticmethod def header_replace(filename, old_magic, new_magic, inplace=True): From 21bd01ef1677ad52e73b1b4d98ba1364d4c64506 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 2 Dec 2015 02:55:59 +0100 Subject: [PATCH 120/321] add a --filter option replacing --changed/--unchanged the problem here was that we do not just have changed and unchanged items, but also a lot of items besides regular files which we just back up "as is" without determining whether they are changed or not. thus, we can't support changed/unchanged in a way users would expect them to work. the A/M/U status only applies to the data content of regular files (compared to the index). for all items, we ALWAYS save the metadata, there is no changed / not changed detection there. thus, I replaced this with a --filter option where you can just specify which status chars you want to see listed in the output. E.g. --filter AM will only show regular files with A(dded) or M(odified) state, but nothing else. Not giving --filter defaults to showing all items no matter what status they have. Output is emitted via logger at info level, so it won't show up except if the logger is at that level. --- borg/archiver.py | 17 +++++------------ borg/testsuite/archiver.py | 10 +++++----- docs/usage.rst | 14 +++++++++----- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index eefe8136..47c000ae 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -74,12 +74,8 @@ class Archiver: logger.warning(msg) def print_file_status(self, status, path): - if status == 'U': - if self.unchanged: - print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) - else: - if self.changed: - print("%1s %s" % (status, remove_surrogates(path)), file=sys.stderr) + if self.output_filter is None or status in self.output_filter: + logger.info("%1s %s", status, remove_surrogates(path)) def do_serve(self, args): """Start in server mode. This command is usually not used manually. @@ -128,6 +124,7 @@ class Archiver: def do_create(self, args): """Create new archive""" + self.output_filter = args.output_filter dry_run = args.dry_run t0 = datetime.now() if not dry_run: @@ -806,10 +803,8 @@ class Archiver: help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: %(default)s""") - subparser.add_argument('--changed', action='store_true', dest='changed', default=False, - help="""display which files were added to the archive""") - subparser.add_argument('--unchanged', action='store_true', dest='unchanged', default=False, - help="""display which files were *not* added to the archive""") + subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', + help='only display items with the given status characters') subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') @@ -1186,8 +1181,6 @@ class Archiver: def run(self, args): os.umask(args.umask) # early, before opening files self.lock_wait = args.lock_wait - self.changed = getattr(args, 'changed', False) - self.unchanged = getattr(args, 'unchanged', False) RemoteRepository.remote_path = args.remote_path RemoteRepository.umask = args.umask setup_logging(level=args.log_level) # do not use loggers before this! diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8542c338..9472cc7c 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -713,11 +713,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago self.create_regular_file('file2', size=1024 * 80) self.cmd('init', self.repository_location) - output = self.cmd('create', '--changed', '--unchanged', self.repository_location + '::test', 'input') + output = self.cmd('create', '-v', self.repository_location + '::test', 'input') self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) # should find first file as unmodified - output = self.cmd('create', '--changed', '--unchanged', self.repository_location + '::test1', 'input') + output = self.cmd('create', '-v', self.repository_location + '::test1', 'input') self.assert_in("U input/file1", output) # this is expected, although surprising, for why, see: # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file @@ -736,15 +736,15 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', self.repository_location + '::test0', 'input') self.assert_not_in('file1', output) # should list the file as unchanged - output = self.cmd('create', '--unchanged', self.repository_location + '::test1', 'input') + output = self.cmd('create', '-v', '--filter=U', self.repository_location + '::test1', 'input') self.assert_in('file1', output) # should *not* list the file as changed - output = self.cmd('create', '--changed', self.repository_location + '::test2', 'input') + output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test2', 'input') self.assert_not_in('file1', output) # change the file self.create_regular_file('file1', size=1024 * 100) # should list the file as changed - output = self.cmd('create', '--changed', self.repository_location + '::test3', 'input') + output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test3', 'input') self.assert_in('file1', output) def test_cmdline_compatibility(self): diff --git a/docs/usage.rst b/docs/usage.rst index 67e6a039..9367791c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -429,10 +429,14 @@ Here are misc. notes about topics that are maybe not covered in enough detail in Item flags ~~~~~~~~~~ -`borg create --changed` outputs a verbose list of all files, directories and other -file system items it considered, with the exception of unchanged files -(for this, also add `--unchanged`). For each item, it prefixes a single-letter -flag that indicates type and/or status of the item. +`borg create -v` outputs a verbose list of all files, directories and other +file system items it considered (no matter whether they had content changes +or not). For each item, it prefixes a single-letter flag that indicates type +and/or status of the item. + +If you are interested only in a subset of that output, you can give e.g. +`--filter=AME` and it will only show regular files with A, M or E status (see +below). A uppercase character represents the status of a regular file relative to the "files" cache (not relative to the repo - this is an issue if the files cache @@ -441,7 +445,7 @@ chunks are stored. For 'U' all data chunks refer to already existing chunks. - 'A' = regular file, added (see also :ref:`a_status_oddity` in the FAQ) - 'M' = regular file, modified -- 'U' = regular file, unchanged (only if `--unchanged` is specified) +- 'U' = regular file, unchanged - 'E' = regular file, an error happened while accessing/reading *this* file A lowercase character means a file type other than a regular file, From d572ceaca22c1e94c9cd78219ad7be4219b9baeb Mon Sep 17 00:00:00 2001 From: Stavros Korokithakis Date: Wed, 2 Dec 2015 16:51:49 +0200 Subject: [PATCH 121/321] Display proper repo URL. --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 1c066d95..f9febd40 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -86,7 +86,7 @@ class Archiver: def do_init(self, args): """Initialize an empty repository""" - logger.info('Initializing repository at "%s"' % args.repository.orig) + logger.info('Initializing repository at "%s"' % args.repository.canonical_path()) repository = self.open_repository(args.repository, create=True, exclusive=True) key = key_creator(repository, args) manifest = Manifest(key, repository) From 887196b00e3cb097eb65e4b4a722c46c0aa9dcee Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 14:14:28 +0100 Subject: [PATCH 122/321] progress indicators: better docstring, minor code improvement --- borg/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 8516496c..813d2277 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -898,6 +898,8 @@ class ProgressIndicatorPercent: :param total: total amount of items :param step: step size in percent :param start: at which percent value to start + :param same_line: if True, emit output always on same line + :param msg: output message, must contain one %f placeholder for the percentage :param file: output file, default: sys.stderr """ self.counter = 0 # 0 .. (total-1) @@ -909,12 +911,10 @@ class ProgressIndicatorPercent: self.same_line = same_line def progress(self, current=None): - if current is None: - current = self.counter - else: + if current is not None: self.counter = current + pct = self.counter * 100 / self.total self.counter += 1 - pct = current * 100 / self.total if pct >= self.trigger_at: self.trigger_at += self.step return pct From df24ce5acd756101e584528c6eeb92003875e2f5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 14:45:16 +0100 Subject: [PATCH 123/321] progress indicators: add tests --- borg/testsuite/helpers.py | 77 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 58861aaa..9556b5e8 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -11,7 +11,8 @@ import msgpack.fallback from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, \ - StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams + StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ + ProgressIndicatorPercent, ProgressIndicatorEndless from . import BaseTestCase, environment_variable, FakeInputs @@ -566,3 +567,77 @@ def test_yes_output(capfd): assert 'intro-msg' in err assert 'retry-msg' not in err assert 'false-msg' in err + + +def test_progress_percentage_multiline(capfd): + pi = ProgressIndicatorPercent(1000, step=5, start=0, same_line=False, msg="%3.0f%%", file=sys.stderr) + pi.show(0) + out, err = capfd.readouterr() + assert err == ' 0%\n' + pi.show(420) + out, err = capfd.readouterr() + assert err == ' 42%\n' + pi.show(1000) + out, err = capfd.readouterr() + assert err == '100%\n' + pi.finish() + out, err = capfd.readouterr() + assert err == '' + + +def test_progress_percentage_sameline(capfd): + pi = ProgressIndicatorPercent(1000, step=5, start=0, same_line=True, msg="%3.0f%%", file=sys.stderr) + pi.show(0) + out, err = capfd.readouterr() + assert err == ' 0%\r' + pi.show(420) + out, err = capfd.readouterr() + assert err == ' 42%\r' + pi.show(1000) + out, err = capfd.readouterr() + assert err == '100%\r' + pi.finish() + out, err = capfd.readouterr() + assert err == ' ' * 4 + '\r' + + +def test_progress_percentage_step(capfd): + pi = ProgressIndicatorPercent(100, step=2, start=0, same_line=False, msg="%3.0f%%", file=sys.stderr) + pi.show() + out, err = capfd.readouterr() + assert err == ' 0%\n' + pi.show() + out, err = capfd.readouterr() + assert err == '' # no output at 1% as we have step == 2 + pi.show() + out, err = capfd.readouterr() + assert err == ' 2%\n' + + +def test_progress_endless(capfd): + pi = ProgressIndicatorEndless(step=1, file=sys.stderr) + pi.show() + out, err = capfd.readouterr() + assert err == '.' + pi.show() + out, err = capfd.readouterr() + assert err == '.' + pi.finish() + out, err = capfd.readouterr() + assert err == '\n' + + +def test_progress_endless_step(capfd): + pi = ProgressIndicatorEndless(step=2, file=sys.stderr) + pi.show() + out, err = capfd.readouterr() + assert err == '' # no output here as we have step == 2 + pi.show() + out, err = capfd.readouterr() + assert err == '.' + pi.show() + out, err = capfd.readouterr() + assert err == '' # no output here as we have step == 2 + pi.show() + out, err = capfd.readouterr() + assert err == '.' From b0975a75b56a646a0641d3146223a5e73ed9381c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 16:58:48 +0100 Subject: [PATCH 124/321] docs: add resources section, with videos, talks, presentations --- docs/index.rst | 1 + docs/resources.rst | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docs/resources.rst diff --git a/docs/index.rst b/docs/index.rst index 210db4a0..c6706685 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -14,6 +14,7 @@ Borg Documentation usage faq support + resources changes internals development diff --git a/docs/resources.rst b/docs/resources.rst new file mode 100644 index 00000000..92cf7822 --- /dev/null +++ b/docs/resources.rst @@ -0,0 +1,31 @@ +.. include:: global.rst.inc +.. _resources: + +Resources +========= + +This is a collection of additional resources that are somehow related to +borgbackup. + + +Videos, Talks, Presentations +---------------------------- + +Some of them refer to attic, but you can do the same stuff (and more) with borgbackup. + +- `BorgBackup Installation and Basic Usage `_ (english screencast) + +- `TW's slides for borgbackup talks / lightning talks `_ (just grab the latest ones) + +- "Attic / Borg Backup" talk from GPN 2015 (video, german audio, english slides): + `media.ccc.de `_ + or + `youtube `_ + +- "Attic" talk from Easterhegg 2015 (video, german audio, english slides): + `media.ccc.de `_ + or + `youtube `_ + +- "Attic Backup: Mount your encrypted backups over ssh", 2014 (video, english): + `youtube `_ From f1f2e78ced8c143f738e5fcf99588b3d7d8915d4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 17:13:39 +0100 Subject: [PATCH 125/321] docs: move related projects to resources section --- README.rst | 10 ++-------- docs/resources.rst | 8 ++++++++ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 35acf491..2777059f 100644 --- a/README.rst +++ b/README.rst @@ -109,7 +109,7 @@ Now doing another backup, just to show off the great deduplication:: This archive: 57.16 MB 46.78 MB 151.67 kB <--- ! All archives: 114.02 MB 93.46 MB 44.81 MB -For a graphical frontend refer to our complementary project `BorgWeb`_. +For a graphical frontend refer to our complementary project `BorgWeb `_. Links ===== @@ -125,18 +125,12 @@ Links * `(Old Mailing List's Archives `_) * `License `_ -Related Projects ----------------- - - * `BorgWeb `_ - * `Atticmatic `_ - * `Attic `_ - Notes ----- Borg is a fork of `Attic`_ and maintained by "`The Borg collective`_". +.. _Attic: https://github.com/jborg/attic .. _The Borg collective: https://borgbackup.readthedocs.org/en/latest/authors.html Differences between Attic and Borg diff --git a/docs/resources.rst b/docs/resources.rst index 92cf7822..88e76dac 100644 --- a/docs/resources.rst +++ b/docs/resources.rst @@ -29,3 +29,11 @@ Some of them refer to attic, but you can do the same stuff (and more) with borgb - "Attic Backup: Mount your encrypted backups over ssh", 2014 (video, english): `youtube `_ + + +Software +-------- + +- `BorgWeb - a very simple web UI for BorgBackup `_ +- some other stuff found at the `BorgBackup Github organisation `_ +- `atticmatic `_ (includes borgmatic) From 1f8b64cc1f13330e30a98ecdc59f3c906904af20 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 17:35:52 +0100 Subject: [PATCH 126/321] readme: add note about clientside encryption --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 2777059f..3d00eaca 100644 --- a/README.rst +++ b/README.rst @@ -57,7 +57,7 @@ Main features **Data encryption** All data can be protected using 256-bit AES encryption, data integrity and - authenticity is verified using HMAC-SHA256. + authenticity is verified using HMAC-SHA256. Data is encrypted clientside. **Compression** All data can be compressed by lz4 (super fast, low compression), zlib From 6d083c06954ca3a4c289d978a1d36eaf84b52c45 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 3 Dec 2015 17:50:37 +0100 Subject: [PATCH 127/321] increase rpc protocol version to 2 this is needed because due to the locking improvements, some rpc calls' signature changed slightly. --- borg/remote.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index 3a6a5e5a..76feecce 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -15,6 +15,8 @@ from .repository import Repository import msgpack +RPC_PROTOCOL_VERSION = 2 + BUFSIZE = 10 * 1024 * 1024 @@ -96,7 +98,7 @@ class RepositoryServer: # pragma: no cover return def negotiate(self, versions): - return 1 + return RPC_PROTOCOL_VERSION def open(self, path, create=False, lock_wait=None, lock=True): path = os.fsdecode(path) @@ -150,10 +152,10 @@ class RemoteRepository: self.x_fds = [self.stdin_fd, self.stdout_fd] try: - version = self.call('negotiate', 1) + version = self.call('negotiate', RPC_PROTOCOL_VERSION) except ConnectionClosed: raise ConnectionClosedWithHint('Is borg working on the server?') - if version != 1: + if version != RPC_PROTOCOL_VERSION: raise Exception('Server insisted on using unsupported protocol version %d' % version) self.id = self.call('open', location.path, create, lock_wait, lock) From 8c91923fb5c8c835abd4179e31ec9d9c77a2540c Mon Sep 17 00:00:00 2001 From: Daniel Dent Date: Sun, 6 Dec 2015 18:08:32 -0800 Subject: [PATCH 128/321] Load over SSL (avoids mixed content) --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3d00eaca..c05f34ee 100644 --- a/README.rst +++ b/README.rst @@ -173,6 +173,6 @@ for the complete license. :alt: Build Status :target: https://travis-ci.org/borgbackup/borg -.. |coverage| image:: http://codecov.io/github/borgbackup/borg/coverage.svg?branch=master +.. |coverage| image:: https://codecov.io/github/borgbackup/borg/coverage.svg?branch=master :alt: Test Coverage - :target: http://codecov.io/github/borgbackup/borg?branch=master + :target: https://codecov.io/github/borgbackup/borg?branch=master From 17952dff4828c284c13f5718b46ba076d6e95412 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 7 Dec 2015 14:17:36 +0100 Subject: [PATCH 129/321] helpers: remove functions that are only used once The read_msgpack and write_msgpack functions were only used in one place each. Since msgpack is read and written in lots of places, having functions with these generic names is confusing. Also, the helpers module is quite a mess, so reducing its size seems to be a good idea. --- borg/helpers.py | 13 ------------- borg/repository.py | 14 ++++++++++---- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 813d2277..df6f1163 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -697,19 +697,6 @@ def location_validator(archive=None): return validator -def read_msgpack(filename): - with open(filename, 'rb') as fd: - return msgpack.unpack(fd) - - -def write_msgpack(filename, d): - with open(filename + '.tmp', 'wb') as fd: - msgpack.pack(d, fd) - fd.flush() - os.fsync(fd.fileno()) - os.rename(filename + '.tmp', filename) - - def decode_dict(d, keys, encoding='utf-8', errors='surrogateescape'): for key in keys: if isinstance(d.get(key), bytes): diff --git a/borg/repository.py b/borg/repository.py index 49e3ac23..80c8ed78 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -10,8 +10,8 @@ import shutil import struct from zlib import crc32 -from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, \ - unhexlify, ProgressIndicatorPercent +import msgpack +from .helpers import Error, ErrorWithTraceback, IntegrityError, unhexlify, ProgressIndicatorPercent from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache @@ -189,7 +189,8 @@ class Repository: else: if do_cleanup: self.io.cleanup(transaction_id) - hints = read_msgpack(os.path.join(self.path, 'hints.%d' % transaction_id)) + with open(os.path.join(self.path, 'hints.%d' % transaction_id), 'rb') as fd: + hints = msgpack.unpack(fd) if hints[b'version'] != 1: raise ValueError('Unknown hints file version: %d' % hints['version']) self.segments = hints[b'segments'] @@ -200,7 +201,12 @@ class Repository: b'segments': self.segments, b'compact': list(self.compact)} transaction_id = self.io.get_segments_transaction_id() - write_msgpack(os.path.join(self.path, 'hints.%d' % transaction_id), hints) + hints_file = os.path.join(self.path, 'hints.%d' % transaction_id) + with open(hints_file + '.tmp', 'wb') as fd: + msgpack.pack(hints, fd) + fd.flush() + os.fsync(fd.fileno()) + os.rename(hints_file + '.tmp', hints_file) self.index.write(os.path.join(self.path, 'index.tmp')) os.rename(os.path.join(self.path, 'index.tmp'), os.path.join(self.path, 'index.%d' % transaction_id)) From 3ca1d33d5c379d1c010f20fd47972381a320fc51 Mon Sep 17 00:00:00 2001 From: Jakob Schnitzer Date: Mon, 7 Dec 2015 18:40:06 +0100 Subject: [PATCH 130/321] fix format of umask in help pages fixes #463 --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 8b4e3c1a..51801dd2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -675,7 +675,7 @@ class Archiver: common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', help='do not load/update the file metadata cache used to detect unchanged files') common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M', - help='set umask to M (local and remote, default: %(default)s)') + help='set umask to M (local and remote, default: %(default)04o)') common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH', help='set remote path to executable (default: "%(default)s")') From 720fc49498f1dbe8a1b142e93ca3283a6142a806 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 7 Dec 2015 19:13:58 +0100 Subject: [PATCH 131/321] hashindex_add C implementation this was also the loop contents of hashindex_merge, but we also need it callable from Cython/Python code. this saves some cycles, esp. if the key is already present in the index. --- borg/_hashindex.c | 21 ++++++++++++--------- borg/cache.py | 13 +++---------- borg/hashindex.pyx | 9 +++++++++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index a61c644b..16adbdfc 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -390,21 +390,24 @@ hashindex_summarize(HashIndex *index, long long *total_size, long long *total_cs *total_chunks = chunks; } +static void +hashindex_add(HashIndex *index, const void *key, int32_t *other_values) +{ + int32_t *my_values = (int32_t *)hashindex_get(index, key); + if(my_values == NULL) { + hashindex_set(index, key, other_values); + } else { + *my_values += *other_values; + } +} + static void hashindex_merge(HashIndex *index, HashIndex *other) { int32_t key_size = index->key_size; - const int32_t *other_values; - int32_t *my_values; void *key = NULL; while((key = hashindex_next_key(other, key))) { - other_values = key + key_size; - my_values = (int32_t *)hashindex_get(index, key); - if(my_values == NULL) { - hashindex_set(index, key, other_values); - } else { - *my_values += *other_values; - } + hashindex_add(index, key, key + key_size); } } diff --git a/borg/cache.py b/borg/cache.py index c911283d..eaefc990 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -255,18 +255,11 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" for id in ids: os.unlink(mkpath(id)) - def add(chunk_idx, id, size, csize, incr=1): - try: - count, size, csize = chunk_idx[id] - chunk_idx[id] = count + incr, size, csize - except KeyError: - chunk_idx[id] = incr, size, csize - def fetch_and_build_idx(archive_id, repository, key): chunk_idx = ChunkIndex() cdata = repository.get(archive_id) data = key.decrypt(archive_id, cdata) - add(chunk_idx, archive_id, len(data), len(cdata)) + chunk_idx.add(archive_id, 1, len(data), len(cdata)) archive = msgpack.unpackb(data) if archive[b'version'] != 1: raise Exception('Unknown archive metadata version') @@ -274,7 +267,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" unpacker = msgpack.Unpacker() for item_id, chunk in zip(archive[b'items'], repository.get_many(archive[b'items'])): data = key.decrypt(item_id, chunk) - add(chunk_idx, item_id, len(data), len(chunk)) + chunk_idx.add(item_id, 1, len(data), len(chunk)) unpacker.feed(data) for item in unpacker: if not isinstance(item, dict): @@ -282,7 +275,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" continue if b'chunks' in item: for chunk_id, size, csize in item[b'chunks']: - add(chunk_idx, chunk_id, size, csize) + chunk_idx.add(chunk_id, 1, size, csize) if self.do_cache: fn = mkpath(archive_id) fn_tmp = mkpath(archive_id, suffix='.tmp') diff --git a/borg/hashindex.pyx b/borg/hashindex.pyx index 0b4dc260..5fc8d6e4 100644 --- a/borg/hashindex.pyx +++ b/borg/hashindex.pyx @@ -15,6 +15,7 @@ cdef extern from "_hashindex.c": long long *unique_size, long long *unique_csize, long long *total_unique_chunks, long long *total_chunks) void hashindex_merge(HashIndex *index, HashIndex *other) + void hashindex_add(HashIndex *index, void *key, void *value) int hashindex_get_size(HashIndex *index) int hashindex_write(HashIndex *index, char *path) void *hashindex_get(HashIndex *index, void *key) @@ -196,6 +197,14 @@ cdef class ChunkIndex(IndexBase): &total_unique_chunks, &total_chunks) return total_size, total_csize, unique_size, unique_csize, total_unique_chunks, total_chunks + def add(self, key, refs, size, csize): + assert len(key) == self.key_size + cdef int[3] data + data[0] = _htole32(refs) + data[1] = _htole32(size) + data[2] = _htole32(csize) + hashindex_add(self.index, key, data) + def merge(self, ChunkIndex other): hashindex_merge(self.index, other.index) From 68225af4497ae53afdde7a25cc68ea28dc812c51 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 00:21:46 +0100 Subject: [PATCH 132/321] archive checker: remove report_progress, fix log levels --- borg/archive.py | 34 ++++++++++++++++++---------------- borg/archiver.py | 1 + borg/testsuite/archiver.py | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index d5b46792..f4d98a3a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -3,7 +3,6 @@ from datetime import datetime from getpass import getuser from itertools import groupby import errno -import logging from .logger import create_logger logger = create_logger() @@ -663,20 +662,24 @@ class ArchiveChecker: self.possibly_superseded = set() def check(self, repository, repair=False, archive=None, last=None): - self.report_progress('Starting archive consistency check...') + logger.info('Starting archive consistency check...') self.check_all = archive is None and last is None self.repair = repair self.repository = repository self.init_chunks() self.key = self.identify_key(repository) if Manifest.MANIFEST_ID not in self.chunks: + logger.error("Repository manifest not found!") + self.error_found = True self.manifest = self.rebuild_manifest() else: self.manifest, _ = Manifest.load(repository, key=self.key) self.rebuild_refcounts(archive=archive, last=last) self.orphan_chunks_check() self.finish() - if not self.error_found: + if self.error_found: + logger.error('Archive consistency check complete, problems found.') + else: logger.info('Archive consistency check complete, no problems found.') return self.repair or not self.error_found @@ -696,11 +699,6 @@ class ArchiveChecker: for id_ in result: self.chunks[id_] = (0, 0, 0) - def report_progress(self, msg, error=False): - if error: - self.error_found = True - logger.log(logging.ERROR if error else logging.WARNING, msg) - def identify_key(self, repository): cdata = repository.get(next(self.chunks.iteritems())[0]) return key_factory(repository, cdata) @@ -710,7 +708,7 @@ class ArchiveChecker: Iterates through all objects in the repository looking for archive metadata blocks. """ - self.report_progress('Rebuilding missing manifest, this might take some time...', error=True) + logger.info('Rebuilding missing manifest, this might take some time...') manifest = Manifest(self.key, self.repository) for chunk_id, _ in self.chunks.iteritems(): cdata = self.repository.get(chunk_id) @@ -727,9 +725,9 @@ class ArchiveChecker: except (TypeError, ValueError, StopIteration): continue if isinstance(archive, dict) and b'items' in archive and b'cmdline' in archive: - self.report_progress('Found archive ' + archive[b'name'].decode('utf-8'), error=True) + logger.info('Found archive %s', archive[b'name'].decode('utf-8')) manifest.archives[archive[b'name'].decode('utf-8')] = {b'id': chunk_id, b'time': archive[b'time']} - self.report_progress('Manifest rebuild complete', error=True) + logger.info('Manifest rebuild complete.') return manifest def rebuild_refcounts(self, archive=None, last=None): @@ -771,7 +769,8 @@ class ArchiveChecker: for chunk_id, size, csize in item[b'chunks']: if chunk_id not in self.chunks: # If a file chunk is missing, create an all empty replacement chunk - self.report_progress('{}: Missing file chunk detected (Byte {}-{})'.format(item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size), error=True) + logger.error('{}: Missing file chunk detected (Byte {}-{})'.format(item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size)) + self.error_found = True data = bytes(size) chunk_id = self.key.id_hash(data) cdata = self.key.encrypt(data) @@ -800,7 +799,8 @@ class ArchiveChecker: def report(msg, chunk_id, chunk_no): cid = hexlify(chunk_id).decode('ascii') msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items - self.report_progress(msg, error=True) + self.error_found = True + logger.error(msg) i = 0 for state, items in groupby(archive[b'items'], missing_chunk_detector): @@ -841,7 +841,8 @@ class ArchiveChecker: logger.info('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives)) archive_id = info[b'id'] if archive_id not in self.chunks: - self.report_progress('Archive metadata block is missing', error=True) + logger.error('Archive metadata block is missing!') + self.error_found = True del self.manifest.archives[name] continue mark_as_possibly_superseded(archive_id) @@ -876,12 +877,13 @@ class ArchiveChecker: unused.add(id_) orphaned = unused - self.possibly_superseded if orphaned: - self.report_progress('{} orphaned objects found'.format(len(orphaned)), error=True) + logger.error('{} orphaned objects found!'.format(len(orphaned))) + self.error_found = True if self.repair: for id_ in unused: self.repository.delete(id_) else: - self.report_progress('Orphaned objects check skipped (needs all archives checked)') + logger.warning('Orphaned objects check skipped (needs all archives checked).') def finish(self): if self.repair: diff --git a/borg/archiver.py b/borg/archiver.py index 47c000ae..fe9813d2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -109,6 +109,7 @@ class Archiver: if repository.check(repair=args.repair): logger.info('Repository check complete, no problems found.') else: + logger.error('Repository check complete, problems found.') return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( repository, repair=args.repair, archive=args.repository.archive, last=args.last): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 9472cc7c..8e24671b 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -992,7 +992,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): repository.delete(Manifest.MANIFEST_ID) repository.commit() self.cmd('check', self.repository_location, exit_code=1) - output = self.cmd('check', '--repair', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) self.assert_in('archive1', output) self.assert_in('archive2', output) self.cmd('check', self.repository_location, exit_code=0) From cb821b119b43ac940ff6960c1c8140f30de8f6c8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 01:37:34 +0100 Subject: [PATCH 133/321] remove --log-level, add --debug and --info option, update docs removed --log-level due to overlap with how --verbose works now. for consistency, added --info as alias to --verbose (as the effect is setting INFO log level). also added --debug which sets DEBUG log level. note: there are no messages emitted at DEBUG level yet. WARNING is the default (because we want mostly silent behaviour, except if something serious happens), so we don't need --warning as an option. --- borg/archiver.py | 10 +++++----- borg/testsuite/archiver.py | 6 +++--- docs/changes.rst | 4 +++- docs/usage.rst | 16 ++++++++++++---- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index fe9813d2..39ecaaa2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -664,12 +664,12 @@ class Archiver: def build_parser(self, args=None, prog=None): common_parser = argparse.ArgumentParser(add_help=False, prog=prog) - common_parser.add_argument('-v', '--verbose', dest='log_level', + common_parser.add_argument('-v', '--verbose', '--info', dest='log_level', action='store_const', const='info', default='warning', - help='verbose output, same as --log-level=info') - common_parser.add_argument('--log-level', dest='log_level', default='warning', metavar='LEVEL', - choices=('debug', 'info', 'warning', 'error', 'critical'), - help='set the log level to LEVEL, default: %(default)s)') + help='enable informative (verbose) output, work on log level INFO') + common_parser.add_argument('--debug', dest='log_level', + action='store_const', const='debug', default='warning', + help='enable debug output, work on log level DEBUG') common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, help='wait for the lock, but max. N seconds (default: %(default)d).') common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8e24671b..0698befc 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -950,13 +950,13 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): return archive, repository def test_check_usage(self): - output = self.cmd('check', '--log-level=info', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) - output = self.cmd('check', '--log-level=info', '--repository-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) self.assert_not_in('Starting archive consistency check', output) - output = self.cmd('check', '--log-level=info', '--archives-only', self.repository_location, exit_code=0) + output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0) self.assert_not_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) diff --git a/docs/changes.rst b/docs/changes.rst index d87dba7d..712e8998 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -25,7 +25,9 @@ New features: - implement borg break-lock command, fixes #157 - include system info below traceback, fixes #324 - use ISO-8601 date and time format, fixes #375 -- add --log-level to set the level of the builtin logging configuration, fixes #426 +- add --debug and --info (same as --verbose) to set the log level of the + builtin logging configuration (which otherwise defaults to warning), + fixes #426 - configure logging via env var BORG_LOGGING_CONF - add a --no-progress flag to forcibly disable progress info diff --git a/docs/usage.rst b/docs/usage.rst index 9367791c..1d31d6e5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -14,12 +14,20 @@ General Type of log output ~~~~~~~~~~~~~~~~~~ -You can set the log level of the builtin logging configuration using the ---log-level option. +The log level of the builtin logging configuration defaults to WARNING. +This is because we want |project_name| to be mostly silent and only output +warnings (plus errors and critical messages). -Supported levels: ``debug``, ``info``, ``warning``, ``error``, ``critical``. +Use --verbose or --info to set INFO (you will get informative output then +additionally to warnings, errors, critical messages). +Use --debug to set DEBUG to get output made for debugging. -All log messages created with at least the given level will be output. +All log messages created with at least the set level will be output. + +Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL + +While you can set misc. log levels, do not expect that every command will +give different output on different log levels - it's just a possibility. Return codes ~~~~~~~~~~~~ From 74ee8154f22e608c3972e638cba50dd05a626cae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 01:45:22 +0100 Subject: [PATCH 134/321] add developer docs about output and logging --- docs/development.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/development.rst b/docs/development.rst index 753f3d55..0d75dfe4 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -19,6 +19,18 @@ separate sections either. The `flake8 `_ commandline tool should be used to check for style errors before sending pull requests. +Output and Logging +------------------ +When writing logger calls, always use correct log level (debug only for +debugging, info for informative messages, warning for warnings, error for +errors, critical for critical errors/states). + +When directly talking to the user (e.g. Y/N questions), do not use logging, +but directly output to stderr (not: stdout, it could be connected to a pipe). + +To control the amount and kinds of messages output to stderr or emitted at +info level, use flags like --stats. + Building a development environment ---------------------------------- From 9c3b206bf792e23f49b89b66ef9003700a915725 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 10:12:15 +0100 Subject: [PATCH 135/321] fix mailing list address in setup.py, fixes #468 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cf699447..c3ed123f 100644 --- a/setup.py +++ b/setup.py @@ -224,7 +224,7 @@ setup( 'write_to': 'borg/_version.py', }, author='The Borg Collective (see AUTHORS file)', - author_email='borgbackup@librelist.com', + author_email='borgbackup@python.org', url='https://borgbackup.readthedocs.org/', description='Deduplicated, encrypted, authenticated and compressed backups', long_description=long_description, From f7abff87f958b7c8967cb02ad31eda577afb0f48 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 10:21:20 +0100 Subject: [PATCH 136/321] ran build_usage --- docs/usage/break-lock.rst.inc | 34 +++++++++++++++++++++ docs/usage/change-passphrase.rst.inc | 23 ++++++++------ docs/usage/check.rst.inc | 14 ++++++--- docs/usage/create.rst.inc | 17 ++++++++--- docs/usage/debug-delete-obj.rst.inc | 27 +++++++++------- docs/usage/debug-dump-archive-items.rst.inc | 23 ++++++++------ docs/usage/debug-get-obj.rst.inc | 28 +++++++++-------- docs/usage/debug-put-obj.rst.inc | 26 +++++++++------- docs/usage/delete.rst.inc | 29 ++++++++++-------- docs/usage/extract.rst.inc | 17 +++++++---- docs/usage/info.rst.inc | 24 +++++++++------ docs/usage/init.rst.inc | 13 +++++--- docs/usage/list.rst.inc | 13 +++++--- docs/usage/mount.rst.inc | 13 +++++--- docs/usage/prune.rst.inc | 16 ++++++---- docs/usage/rename.rst.inc | 26 +++++++++------- docs/usage/serve.rst.inc | 13 +++++--- docs/usage/upgrade.rst.inc | 31 +++++++++++-------- 18 files changed, 251 insertions(+), 136 deletions(-) create mode 100644 docs/usage/break-lock.rst.inc diff --git a/docs/usage/break-lock.rst.inc b/docs/usage/break-lock.rst.inc new file mode 100644 index 00000000..d59b1dc0 --- /dev/null +++ b/docs/usage/break-lock.rst.inc @@ -0,0 +1,34 @@ +.. _borg_break-lock: + +borg break-lock +--------------- +:: + + usage: borg break-lock [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + REPOSITORY + + Break the repository lock (e.g. in case it was left by a dead borg. + + positional arguments: + REPOSITORY repository for which to break the locks + + optional arguments: + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command breaks the repository and cache locks. +Please use carefully and only while no borg process (on any machine) is +trying to access the Cache or the Repository. diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index 9d0926d1..eb52399c 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -4,8 +4,9 @@ borg change-passphrase ---------------------- :: - usage: borg change-passphrase [-h] [-v] [--show-rc] [--no-files-cache] - [--umask M] [--remote-path PATH] + usage: borg change-passphrase [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] + [--remote-path PATH] [REPOSITORY] Change repository key file passphrase @@ -14,13 +15,17 @@ borg change-passphrase REPOSITORY optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index f2a05956..fb15b428 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -4,9 +4,9 @@ borg check ---------- :: - usage: borg check [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [--repository-only] [--archives-only] - [--repair] [--last N] + usage: borg check [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + [--repository-only] [--archives-only] [--repair] [--last N] [REPOSITORY_OR_ARCHIVE] Check repository consistency @@ -17,11 +17,15 @@ borg check optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") --repository-only only perform repository checks --archives-only only perform archives checks diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index c52108cb..f0068574 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -4,8 +4,9 @@ borg create ----------- :: - usage: borg create [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-s] [-p] [-e PATTERN] + usage: borg create [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-s] + [-p] [--filter STATUSCHARS] [-e PATTERN] [--exclude-from EXCLUDEFILE] [--exclude-caches] [--exclude-if-present FILENAME] [--keep-tag-files] [-c SECONDS] [-x] [--numeric-owner] @@ -23,17 +24,23 @@ borg create optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -s, --stats print statistics for the created archive - -p, --progress toggle progress display while creating the archive, + -p, --progress, --no-progress + toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: True + --filter STATUSCHARS only display items with the given status characters -e PATTERN, --exclude PATTERN exclude paths matching PATTERN --exclude-from EXCLUDEFILE diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc index 3cdea321..b02d7b72 100644 --- a/docs/usage/debug-delete-obj.rst.inc +++ b/docs/usage/debug-delete-obj.rst.inc @@ -4,24 +4,29 @@ borg debug-delete-obj --------------------- :: - usage: borg debug-delete-obj [-h] [-v] [--show-rc] [--no-files-cache] - [--umask M] [--remote-path PATH] + usage: borg debug-delete-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] + [--remote-path PATH] [REPOSITORY] IDs [IDs ...] delete the objects with the given IDs from the repo positional arguments: - REPOSITORY repository to use - IDs hex object ID(s) to delete from the repo + REPOSITORY repository to use + IDs hex object ID(s) to delete from the repo optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc index cb68493d..9265f2c0 100644 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ b/docs/usage/debug-dump-archive-items.rst.inc @@ -4,23 +4,28 @@ borg debug-dump-archive-items ----------------------------- :: - usage: borg debug-dump-archive-items [-h] [-v] [--show-rc] [--no-files-cache] + usage: borg debug-dump-archive-items [-h] [-v] [--debug] [--lock-wait N] + [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] ARCHIVE dump (decrypted, decompressed) archive items metadata (not: data) positional arguments: - ARCHIVE archive to dump + ARCHIVE archive to dump optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc index 8343d866..f3213152 100644 --- a/docs/usage/debug-get-obj.rst.inc +++ b/docs/usage/debug-get-obj.rst.inc @@ -4,25 +4,29 @@ borg debug-get-obj ------------------ :: - usage: borg debug-get-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] + usage: borg debug-get-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] ID PATH get object contents from the repository and write it into file positional arguments: - REPOSITORY repository to use - ID hex object ID to get from the repo - PATH file to write object data into + REPOSITORY repository to use + ID hex object ID to get from the repo + PATH file to write object data into optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc index cb14b5de..44767c27 100644 --- a/docs/usage/debug-put-obj.rst.inc +++ b/docs/usage/debug-put-obj.rst.inc @@ -4,24 +4,28 @@ borg debug-put-obj ------------------ :: - usage: borg debug-put-obj [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] + usage: borg debug-put-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] PATH [PATH ...] put file(s) contents into the repository positional arguments: - REPOSITORY repository to use - PATH file(s) to read and create object(s) from + REPOSITORY repository to use + PATH file(s) to read and create object(s) from optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index afdbd9db..587e7064 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -4,25 +4,30 @@ borg delete ----------- :: - usage: borg delete [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-s] [-c] + usage: borg delete [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-s] + [-c] [TARGET] Delete an existing repository or archive positional arguments: - TARGET archive or repository to delete + TARGET archive or repository to delete optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") - -s, --stats print statistics for the deleted archive - -c, --cache-only delete only the local cache for the given repository + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + -s, --stats print statistics for the deleted archive + -c, --cache-only delete only the local cache for the given repository Description ~~~~~~~~~~~ diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index cbacd703..62c45d46 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -4,10 +4,11 @@ borg extract ------------ :: - usage: borg extract [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-n] [-e PATTERN] - [--exclude-from EXCLUDEFILE] [--numeric-owner] - [--strip-components NUMBER] [--stdout] [--sparse] + usage: borg extract [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-n] + [-e PATTERN] [--exclude-from EXCLUDEFILE] + [--numeric-owner] [--strip-components NUMBER] [--stdout] + [--sparse] ARCHIVE [PATH [PATH ...]] Extract archive contents @@ -18,11 +19,15 @@ borg extract optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -n, --dry-run do not actually change any files -e PATTERN, --exclude PATTERN diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index 2ddb651e..7c2c44b5 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -4,23 +4,27 @@ borg info --------- :: - usage: borg info [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] + usage: borg info [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] ARCHIVE Show archive details such as disk space used positional arguments: - ARCHIVE archive to display information about + ARCHIVE archive to display information about optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index 4e0f47b0..c4e48284 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -4,8 +4,9 @@ borg init --------- :: - usage: borg init [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-e {none,keyfile,repokey,passphrase}] + usage: borg init [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + [-e {none,keyfile,repokey,passphrase}] [REPOSITORY] Initialize an empty repository @@ -15,11 +16,15 @@ borg init optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -e {none,keyfile,repokey,passphrase}, --encryption {none,keyfile,repokey,passphrase} select encryption key mode diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 882fff25..fd435501 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -4,8 +4,9 @@ borg list --------- :: - usage: borg list [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [--short] [-p PREFIX] + usage: borg list [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [--short] + [-p PREFIX] [REPOSITORY_OR_ARCHIVE] List archive or repository contents @@ -16,11 +17,15 @@ borg list optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") --short only print file/directory names, nothing else -p PREFIX, --prefix PREFIX diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index 0407a095..380df549 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -4,8 +4,9 @@ borg mount ---------- :: - usage: borg mount [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-f] [-o OPTIONS] + usage: borg mount [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-f] + [-o OPTIONS] REPOSITORY_OR_ARCHIVE MOUNTPOINT Mount archive or an entire repository as a FUSE fileystem @@ -17,11 +18,15 @@ borg mount optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -f, --foreground stay in foreground, do not daemonize -o OPTIONS Extra mount options diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index c2133736..08504b5d 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -4,10 +4,10 @@ borg prune ---------- :: - usage: borg prune [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-n] [-s] [--keep-within WITHIN] - [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY] - [-p PREFIX] + usage: borg prune [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-n] + [-s] [--keep-within WITHIN] [-H HOURLY] [-d DAILY] + [-w WEEKLY] [-m MONTHLY] [-y YEARLY] [-p PREFIX] [REPOSITORY] Prune repository archives according to specified rules @@ -17,11 +17,15 @@ borg prune optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -n, --dry-run do not change repository -s, --stats print statistics for the deleted archive diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc index 23cb338e..8e0a4b61 100644 --- a/docs/usage/rename.rst.inc +++ b/docs/usage/rename.rst.inc @@ -4,24 +4,28 @@ borg rename ----------- :: - usage: borg rename [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] + usage: borg rename [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] ARCHIVE NEWNAME Rename an existing archive positional arguments: - ARCHIVE archive to rename - NEWNAME the new archive name to use + ARCHIVE archive to rename + NEWNAME the new archive name to use optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") Description ~~~~~~~~~~~ diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index 8c405226..1e29ff2a 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -4,19 +4,24 @@ borg serve ---------- :: - usage: borg serve [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [--restrict-to-path PATH] + usage: borg serve [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + [--restrict-to-path PATH] Start in server mode. This command is usually not used manually. optional arguments: -h, --help show this help message and exit - -v, --verbose verbose output + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to detect unchanged files - --umask M set umask to M (local and remote, default: 63) + --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") --restrict-to-path PATH restrict repository access to PATH diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index 7b9660b5..5aadcd3e 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -4,26 +4,31 @@ borg upgrade ------------ :: - usage: borg upgrade [-h] [-v] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-n] [-i] + usage: borg upgrade [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] [-n] + [-i] [REPOSITORY] upgrade a repository from a previous version positional arguments: - REPOSITORY path to the repository to be upgraded + REPOSITORY path to the repository to be upgraded optional arguments: - -h, --help show this help message and exit - -v, --verbose verbose output - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 63) - --remote-path PATH set remote path to executable (default: "borg") - -n, --dry-run do not change repository - -i, --inplace rewrite repository in place, with no chance of going - back to older versions of the repository. + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + -n, --dry-run do not change repository + -i, --inplace rewrite repository in place, with no chance of going + back to older versions of the repository. Description ~~~~~~~~~~~ From 41eab542a842be74bf5b133deb16a4655099fb58 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 10:33:27 +0100 Subject: [PATCH 137/321] add borg upgrade to the docs, fixes #464 --- docs/usage.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index 1d31d6e5..ceea1cf5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -413,6 +413,15 @@ Examples command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...] +.. include:: usage/upgrade.rst.inc + +Examples +~~~~~~~~ +:: + + borg upgrade -v /mnt/backup + + Miscellaneous Help ------------------ From f97b9eb90da90248cd1726a55e4c18ec52dad2aa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 12:16:58 +0100 Subject: [PATCH 138/321] updated CHANGES --- docs/changes.rst | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 712e8998..8c2c3b0e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -7,7 +7,8 @@ Version 0.29.0 Compatibility notes: - when upgrading to 0.29.0 you need to upgrade client as well as server - installations due to the locking related changes. + installations due to the locking related changes otherwise you'll get an + error msg about a RPC protocol mismatch. - the default waiting time for a lock changed from infinity to 1 second for a better interactive user experience. if the repo you want to access is currently locked, borg will now terminate after 1s with an error message. @@ -16,8 +17,12 @@ Compatibility notes: Bug fixes: +- hash table tuning (better chosen hashtable load factor 0.75 and prime initial + size of 1031 gave ~1000x speedup in some scenarios) - avoid creation of an orphan lock for one case, see #285 - --keep-tag-files: fix file mode and multiple tag files in one directory, #432 +- fix format of umask in help pages, #463 +- borg init: display proper repo URL New features: @@ -28,22 +33,34 @@ New features: - add --debug and --info (same as --verbose) to set the log level of the builtin logging configuration (which otherwise defaults to warning), fixes #426 + note: there are no messages emitted at DEBUG level currently. - configure logging via env var BORG_LOGGING_CONF - add a --no-progress flag to forcibly disable progress info +- add a --filter option for status characters: e.g. to show only the added + or modified files (and also errors), use "borg create -v --filter=AME ...". +- more progress indicators, fixes #394 Other changes: -- fix progress tests on travis +- hashindex_add C implementation (speed up cache resync for new archives) +- increase rpc protocol version to 2, fixes #458 +- silence borg by default (via log level WARNING) - get rid of C compiler warnings, fixes #391 - upgrade OS X FUSE to 3.0.9 on the OS X binary build system - docs: - - document new mailing list borgbackup@python.org + - new mailing list borgbackup@python.org, #468 - readthedocs: color and logo improvements + - load coverage icons over SSL (avoids mixed content) - more precise binary installation steps - update release procedure docs about OS X FUSE - FAQ entry about unexpected 'A' status for unchanged file(s), fixes #403 - add docs about 'E' file status + - add "borg upgrade" docs, fixes #464 + - add developer docs about output and logging + - clarify encryption, add note about clientside encryption + - add resources section, with videos, talks, presentations, #149 + - Borg moved to Arch Linux [community] Version 0.28.2 From 5f1fcb3e638d2b3780265dfd549967452d822578 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 17:47:00 +0100 Subject: [PATCH 139/321] add hint "not released yet" to latest changelog entry --- docs/changes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 8c2c3b0e..aa7dc7e6 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,8 +1,8 @@ Changelog ========= -Version 0.29.0 --------------- +Version 0.29.0 (not released yet) +--------------------------------- Compatibility notes: From 0c076ad114206a370d129eae3e6771e856cb1d67 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 18 Nov 2015 02:27:25 +0100 Subject: [PATCH 140/321] compact_segments: save_space -> free unused segments quickly as soon as one target segment is full, it is a good time to commit it and remove the source segments that are already completely unused (because they were transferred int the target segment). so, for compact_segments(save_space=True), the additional space needed should be about 1 segment size. note: we can't just do that at the end of one source segment as this might create very small target segments, which is not wanted. --- borg/archive.py | 8 ++--- borg/archiver.py | 18 ++++++++--- borg/remote.py | 8 ++--- borg/repository.py | 58 +++++++++++++++++++++++++----------- borg/testsuite/repository.py | 2 +- 5 files changed, 64 insertions(+), 30 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index f4d98a3a..011d7845 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -661,7 +661,7 @@ class ArchiveChecker: self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, archive=None, last=None): + def check(self, repository, repair=False, archive=None, last=None, save_space=False): logger.info('Starting archive consistency check...') self.check_all = archive is None and last is None self.repair = repair @@ -676,7 +676,7 @@ class ArchiveChecker: self.manifest, _ = Manifest.load(repository, key=self.key) self.rebuild_refcounts(archive=archive, last=last) self.orphan_chunks_check() - self.finish() + self.finish(save_space=save_space) if self.error_found: logger.error('Archive consistency check complete, problems found.') else: @@ -885,7 +885,7 @@ class ArchiveChecker: else: logger.warning('Orphaned objects check skipped (needs all archives checked).') - def finish(self): + def finish(self, save_space=False): if self.repair: self.manifest.write() - self.repository.commit() + self.repository.commit(save_space=save_space) diff --git a/borg/archiver.py b/borg/archiver.py index fcccaccf..187788d2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -105,10 +105,11 @@ class Archiver: env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): return EXIT_ERROR if not args.archives_only: - if not repository.check(repair=args.repair): + if not repository.check(repair=args.repair, save_space=args.save_space): return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( - repository, repair=args.repair, archive=args.repository.archive, last=args.last): + repository, repair=args.repair, archive=args.repository.archive, + last=args.last, save_space=args.save_space): return EXIT_WARNING return EXIT_SUCCESS @@ -332,7 +333,7 @@ class Archiver: stats = Statistics() archive.delete(stats) manifest.write() - repository.commit() + repository.commit(save_space=args.save_space) cache.commit() if args.stats: logger.info(stats.summary.format(label='Deleted data:', stats=stats)) @@ -487,7 +488,7 @@ class Archiver: Archive(repository, key, manifest, archive.name, cache).delete(stats) if to_delete and not args.dry_run: manifest.write() - repository.commit() + repository.commit(save_space=args.save_space) cache.commit() if args.stats: logger.info(stats.summary.format(label='Deleted data:', stats=stats)) @@ -762,6 +763,9 @@ class Archiver: subparser.add_argument('--repair', dest='repair', action='store_true', default=False, help='attempt to repair any inconsistencies found') + subparser.add_argument('--save-space', dest='save_space', action='store_true', + default=False, + help='work slower, but using less space') subparser.add_argument('--last', dest='last', type=int, default=None, metavar='N', help='only check last N archives (Default: all)') @@ -926,6 +930,9 @@ class Archiver: subparser.add_argument('-c', '--cache-only', dest='cache_only', action='store_true', default=False, help='delete only the local cache for the given repository') + subparser.add_argument('--save-space', dest='save_space', action='store_true', + default=False, + help='work slower, but using less space') subparser.add_argument('target', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') @@ -1043,6 +1050,9 @@ class Archiver: help='number of yearly archives to keep') subparser.add_argument('-p', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') + subparser.add_argument('--save-space', dest='save_space', action='store_true', + default=False, + help='work slower, but using less space') subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to prune') diff --git a/borg/remote.py b/borg/remote.py index 76feecce..f724b80d 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -273,11 +273,11 @@ class RemoteRepository: w_fds = [] self.ignore_responses |= set(waiting_for) - def check(self, repair=False): - return self.call('check', repair) + def check(self, repair=False, save_space=False): + return self.call('check', repair, save_space) - def commit(self, *args): - return self.call('commit') + def commit(self, save_space=False): + return self.call('commit', save_space) def rollback(self, *args): return self.call('rollback') diff --git a/borg/repository.py b/borg/repository.py index 3765d9c5..36cd0f66 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -158,11 +158,11 @@ class Repository: self.lock.release() self.lock = None - def commit(self): + def commit(self, save_space=False): """Commit transaction """ self.io.write_commit() - self.compact_segments() + self.compact_segments(save_space=save_space) self.write_index() self.rollback() @@ -220,31 +220,50 @@ class Repository: os.unlink(os.path.join(self.path, name)) self.index = None - def compact_segments(self): + def compact_segments(self, save_space=False): """Compact sparse segments by copying data into new segments """ if not self.compact: return index_transaction_id = self.get_index_transaction_id() segments = self.segments + unused = [] # list of segments, that are not used anymore + + def complete_xfer(): + # complete the transfer (usually exactly when some target segment + # is full, or at the very end when everything is processed) + nonlocal unused + # commit the new, compact, used segments + self.io.write_commit() + # get rid of the old, sparse, unused segments. free space. + for segment in unused: + assert self.segments.pop(segment) == 0 + self.io.delete_segment(segment) + unused = [] + for segment in sorted(self.compact): if self.io.segment_exists(segment): for tag, key, offset, data in self.io.iter_objects(segment, include_data=True): if tag == TAG_PUT and self.index.get(key, (-1, -1)) == (segment, offset): - new_segment, offset = self.io.write_put(key, data) + try: + new_segment, offset = self.io.write_put(key, data, raise_full=save_space) + except LoggedIO.SegmentFull: + complete_xfer() + new_segment, offset = self.io.write_put(key, data) self.index[key] = new_segment, offset segments.setdefault(new_segment, 0) segments[new_segment] += 1 segments[segment] -= 1 elif tag == TAG_DELETE: if index_transaction_id is None or segment > index_transaction_id: - self.io.write_delete(key) + try: + self.io.write_delete(key, raise_full=save_space) + except LoggedIO.SegmentFull: + complete_xfer() + self.io.write_delete(key) assert segments[segment] == 0 - - self.io.write_commit() - for segment in sorted(self.compact): - assert self.segments.pop(segment) == 0 - self.io.delete_segment(segment) + unused.append(segment) + complete_xfer() self.compact = set() def replay_segments(self, index_transaction_id, segments_transaction_id): @@ -297,7 +316,7 @@ class Repository: if self.segments[segment] == 0: self.compact.add(segment) - def check(self, repair=False): + def check(self, repair=False, save_space=False): """Check repository consistency This method verifies all segment checksums and makes sure @@ -358,7 +377,7 @@ class Repository: if current_index.get(key, (-1, -1)) != value: report_error('Index mismatch for key {}. {} != {}'.format(key, value, current_index.get(key, (-1, -1)))) if repair: - self.compact_segments() + self.compact_segments(save_space=save_space) self.write_index() self.rollback() if error_found: @@ -441,6 +460,9 @@ class Repository: class LoggedIO: + class SegmentFull(Exception): + """raised when a segment is full, before opening next""" + header_fmt = struct.Struct(' self.limit: + if raise_full: + raise self.SegmentFull self.close_segment() if not self._write_fd: if self.segment % self.segments_per_dir == 0: @@ -630,9 +654,9 @@ class LoggedIO: key, data = data[:32], data[32:] return size, tag, key, data - def write_put(self, id, data): + def write_put(self, id, data, raise_full=False): + fd = self.get_write_fd(raise_full=raise_full) size = len(data) + self.put_header_fmt.size - fd = self.get_write_fd() offset = self.offset header = self.header_no_crc_fmt.pack(size, TAG_PUT) crc = self.crc_fmt.pack(crc32(data, crc32(id, crc32(header))) & 0xffffffff) @@ -640,8 +664,8 @@ class LoggedIO: self.offset += size return self.segment, offset - def write_delete(self, id): - fd = self.get_write_fd() + def write_delete(self, id, raise_full=False): + fd = self.get_write_fd(raise_full=raise_full) header = self.header_no_crc_fmt.pack(self.put_header_fmt.size, TAG_DELETE) crc = self.crc_fmt.pack(crc32(id, crc32(header)) & 0xffffffff) fd.write(b''.join((crc, header, id))) diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 713f4402..4094df40 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -311,7 +311,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): # Simulate a crash before compact with patch.object(Repository, 'compact_segments') as compact: self.repository.commit() - compact.assert_called_once_with() + compact.assert_called_once_with(save_space=False) self.reopen() self.check(repair=True) self.assert_equal(self.repository.get(bytes(32)), b'data2') From bec2f72c8e827bdfdb91bfed111fcef564c634c8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 18:22:24 +0100 Subject: [PATCH 141/321] mention --save-space at the place we talk about keeping disk space free --- docs/quickstart.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index b2f30aa0..42201e89 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -17,7 +17,8 @@ a good amount of free space on the filesystem that has your backup repository If you run out of disk space, it can be hard or impossible to free space, because |project_name| needs free space to operate - even to delete backup -archives. +archives. There is a `--save-space` option for some commands, but even with +that |project_name| will need free space to operate. You can use some monitoring process or just include the free space information in your backup log files (you check them regularly anyway, right?). From 499b6f7313bcbb65c810a3409d9af7abdf824012 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 18:27:23 +0100 Subject: [PATCH 142/321] add a test that invokes borg prune --save-space --- borg/testsuite/archiver.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 0698befc..4560f200 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -771,6 +771,21 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_not_in('test1', output) self.assert_in('test2', output) + def test_prune_repository_save_space(self): + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test1', src_dir) + self.cmd('create', self.repository_location + '::test2', src_dir) + output = self.cmd('prune', '-v', '--dry-run', self.repository_location, '--keep-daily=2') + self.assert_in('Keeping archive: test2', output) + self.assert_in('Would prune: test1', output) + output = self.cmd('list', self.repository_location) + self.assert_in('test1', output) + self.assert_in('test2', output) + self.cmd('prune', '--save-space', self.repository_location, '--keep-daily=2') + output = self.cmd('list', self.repository_location) + self.assert_not_in('test1', output) + self.assert_in('test2', output) + def test_prune_repository_prefix(self): self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::foo-2015-08-12-10:00', src_dir) From f1b9b95e0deade9b2260135b169cdd9798d68164 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Tue, 8 Dec 2015 14:30:42 +0100 Subject: [PATCH 143/321] Fix wrong installation instructions for archlinux On arch I don't want to perform a full system upgrade when installing a new package; so let's drop the "yu" part. --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 271fd086..6441db79 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -32,7 +32,7 @@ yet. ============ ===================== ======= Distribution Source Command ============ ===================== ======= -Arch Linux `[community]`_ ``pacman -Syu borg`` +Arch Linux `[community]`_ ``pacman -S borg`` Debian `unstable/sid`_ ``apt install borgbackup`` Ubuntu `Xenial Xerus 16.04`_ ``apt install borgbackup`` OS X `Brew cask`_ ``brew cask install borgbackup`` From 3256f22c74cdd633c7a9d0eda25497a93131a034 Mon Sep 17 00:00:00 2001 From: alex3d Date: Wed, 9 Dec 2015 00:34:25 +0300 Subject: [PATCH 144/321] Optimized fuse inode cache Single-shot unpacker read buffer decreased from (default) 1Mb to 512b. "ls -alR" on ~100k files backup mounted with fuse went from ~7min to 30 seconds. --- borg/fuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/fuse.py b/borg/fuse.py index 19dc09cc..3b2eefa0 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -28,7 +28,7 @@ class ItemCache: def get(self, inode): self.fd.seek(inode - self.offset, io.SEEK_SET) - return next(msgpack.Unpacker(self.fd)) + return next(msgpack.Unpacker(self.fd, read_size=512)) class FuseOperations(llfuse.Operations): From 20fb4c44a016c319b5b2850af027168d22b7451e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 8 Dec 2015 23:44:00 +0100 Subject: [PATCH 145/321] use python 3.5.1 to build binaries --- Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index b7470591..d7eab6f7 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -160,7 +160,7 @@ def install_pythons(boxname) pyenv install 3.3.0 # tests pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests - #pyenv install 3.5.1 # binary build, use latest 3.5.x release + pyenv install 3.5.1 # binary build, use latest 3.5.x release pyenv rehash EOF end @@ -178,8 +178,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.5 release - pyenv global 3.5.0 - pyenv virtualenv 3.5.0 borg-env + pyenv global 3.5.1 + pyenv virtualenv 3.5.1 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end From 458fc747a3171287ec1091601db6c679d8b1075e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 9 Dec 2015 00:55:48 +0100 Subject: [PATCH 146/321] results of some llfuse experiments filed some bug reports, added some notes to Vagrantfile. --- Vagrantfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d7eab6f7..173a3644 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -127,7 +127,8 @@ def packages_netbsd ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/ touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..." mozilla-rootcerts install - # llfuse does not support netbsd + # pkg_add fuse pkg-config # llfuse 0.41.1 supports netbsd, but is still buggy. + # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space pkg_add python34 py34-setuptools ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3 @@ -144,6 +145,7 @@ def install_pyenv(boxname) echo 'eval "$(pyenv init -)"' >> ~/.bash_profile echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile + echo 'export LANG=en_US.UTF-8' >> ~/.bash_profile EOF end @@ -195,7 +197,8 @@ def install_borg(boxname) rm -f borg/*.so borg/*.cpy* rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__ - pip install 'llfuse<0.41' # 0.41 does not install due to UnicodeDecodeError + pip install 'llfuse<0.41' # 0.41.1 throws UnicodeDecodeError at install time: + # https://bitbucket.org/nikratio/python-llfuse/issues/69/unicode-exception-at-install-time pip install -r requirements.d/development.txt pip install -e . EOF From 8671be1ef2f5087b29dc327168335d3f6485b391 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 10:09:06 +0100 Subject: [PATCH 147/321] Increase FUSE read_size to 1024. From https://github.com/borgbackup/borg/pull/480 discussion: Did you try 1024 (linux cache block size) or 4096 (internal sector size of bigger hdds, also used in msgpack fallback.py as lower bound, see link)? I've tested different values - 512 and 1024 are slightly better than 4096 in my case. read_size = 1 ls -laR: 75.57 sec read_size = 64 ls -laR: 27.81 sec read_size = 512 ls -laR: 27.40 sec read_size = 1024 ls -laR: 27.20 sec read_size = 4096 ls -laR: 30.15 sec read_size = 0 ls -laR: 442.96 sec (default) OK, maybe we should go for 1024 then. That happens to be < MTU size, so in case someone works on NFS (or other network FS) we will have less reads, less network packets, less latency. --- borg/fuse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/fuse.py b/borg/fuse.py index 3b2eefa0..448fe02a 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -28,7 +28,7 @@ class ItemCache: def get(self, inode): self.fd.seek(inode - self.offset, io.SEEK_SET) - return next(msgpack.Unpacker(self.fd, read_size=512)) + return next(msgpack.Unpacker(self.fd, read_size=1024)) class FuseOperations(llfuse.Operations): From 34b35761dd48beea847441ac4cf39c50f9cc225a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 10:28:43 +0100 Subject: [PATCH 148/321] remove --progress magic, fixes #476 For 0.29 we worked towards a "silent by default" behaviour, so interactive usage will include -v more frequently in future. But I noticed that this conflicts with the progress display. This would be no problem if users willingly decide which one of --verbose or --progress they want to see, but before this fix, the progress display was activated magically when a tty was detected. So, to counteract this magic, users would need to use --no-progress. That's backwards imho, so I removed the magic again and users have to give --progress when they want to see a progress indicator. Or (alternatively) they give --verbose when they want to see the long file list. --- borg/archiver.py | 7 +++---- borg/testsuite/archiver.py | 27 +-------------------------- 2 files changed, 4 insertions(+), 30 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 187788d2..3583870c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -798,10 +798,9 @@ class Archiver: subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the created archive') - subparser.add_argument('-p', '--progress', '--no-progress', - dest='progress', default=sys.stdin.isatty(), - action=ToggleAction, nargs=0, - help="""toggle progress display while creating the archive, showing Original, + subparser.add_argument('-p', '--progress', dest='progress', + action='store_true', default=False, + help="""show progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: %(default)s""") subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 4560f200..d8ae92cc 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -676,32 +676,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input') self.assert_in("\r", output) # progress forced off - output = self.cmd('create', '--no-progress', self.repository_location + '::test5', 'input') - self.assert_not_in("\r", output) - - @unittest.skipUnless(sys.stdout.isatty(), 'need a tty to test auto-detection') - def test_progress_tty(self): - """test that the --progress and --no-progress flags work, - overriding defaults from the terminal auto-detection""" - self.create_regular_file('file1', size=1024 * 80) - self.cmd('init', self.repository_location) - # without a terminal, no progress expected - output = self.cmd('create', self.repository_location + '::test1', 'input', fork=False) - self.assert_not_in("\r", output) - # with a terminal, progress expected - output = self.cmd('create', self.repository_location + '::test2', 'input', fork=True) - self.assert_in("\r", output) - # without a terminal, progress forced on - output = self.cmd('create', '--progress', self.repository_location + '::test3', 'input', fork=False) - self.assert_in("\r", output) - # with a terminal, progress forced on - output = self.cmd('create', '--progress', self.repository_location + '::test4', 'input', fork=True) - self.assert_in("\r", output) - # without a terminal, progress forced off - output = self.cmd('create', '--no-progress', self.repository_location + '::test5', 'input', fork=False) - self.assert_not_in("\r", output) - # with a terminal, progress forced off - output = self.cmd('create', '--no-progress', self.repository_location + '::test6', 'input', fork=True) + output = self.cmd('create', self.repository_location + '::test5', 'input') self.assert_not_in("\r", output) def test_file_status(self): From e9ab11be497f1195d39503226925e9535bd5d312 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 10:51:56 +0100 Subject: [PATCH 149/321] borg upgrade - fix README contents --- borg/upgrader.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/borg/upgrader.py b/borg/upgrader.py index 81981498..b7e6bf4c 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -50,11 +50,18 @@ class AtticRepositoryUpgrader(Repository): try: self.convert_cache(dryrun) self.convert_segments(segments, dryrun=dryrun, inplace=inplace) + self.borg_readme() finally: self.lock.release() self.lock = None return backup + def borg_readme(self): + readme = os.path.join(self.path, 'README') + os.remove(readme) + with open(readme, 'w') as fd: + fd.write('This is a Borg repository\n') + @staticmethod def convert_segments(segments, dryrun=True, inplace=False): """convert repository segments from attic to borg From 7acda553ffd6ef4383af15c24cb2ddcea5418b6e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:11:06 +0100 Subject: [PATCH 150/321] borg upgrade - fix locking because Repository.__init__ normally opens and locks the repo, and the upgrader just inherited from (borg) Repository, it created a lock file there before the "backup copy" was made. No big problem, but a bit unclean. Fixed it to not lock at the beginning, then make the copy, then lock. --- borg/upgrader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index b7e6bf4c..d4c628e6 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -16,6 +16,10 @@ ATTIC_MAGIC = b'ATTICSEG' class AtticRepositoryUpgrader(Repository): + def __init__(self, *args, **kw): + kw['lock'] = False # do not create borg lock files (now) in attic repo + super().__init__(*args, **kw) + def upgrade(self, dryrun=True, inplace=False): """convert an attic repository to a borg repository @@ -34,8 +38,8 @@ class AtticRepositoryUpgrader(Repository): if not dryrun: shutil.copytree(self.path, backup, copy_function=os.link) logger.info("opening attic repository with borg and converting") - # we need to open the repo to load configuration, keyfiles and segments - self.open(self.path, exclusive=False) + # now lock the repo, after we have made the copy + self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() segments = [filename for i, filename in self.io.segment_iterator()] try: keyfile = self.find_attic_keyfile() From cd804e72b703cda47dd0eadfbe986d49dab85a72 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:35:48 +0100 Subject: [PATCH 151/321] borg upgrade - move some code to convert_repo_index it was a bit confusing to have repo-related code in a method called "convert_cache". also fixed a typo "index index". --- borg/upgrader.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index d4c628e6..4c14c856 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -52,6 +52,7 @@ class AtticRepositoryUpgrader(Repository): self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire() try: + self.convert_repo_index(dryrun) self.convert_cache(dryrun) self.convert_segments(segments, dryrun=dryrun, inplace=inplace) self.borg_readme() @@ -150,8 +151,8 @@ class AtticRepositoryUpgrader(Repository): with open(keyfile, 'w') as f: f.write(data) - def convert_cache(self, dryrun): - """convert caches from attic to borg + def convert_repo_index(self, dryrun): + """convert some repo files those are all hash indexes, so we need to `s/ATTICIDX/BORG_IDX/` in a few locations: @@ -161,6 +162,21 @@ class AtticRepositoryUpgrader(Repository): should probably update, with a lock, see `Repository.open()`, which i'm not sure we should use because it may write data on `Repository.close()`... + """ + transaction_id = self.get_index_transaction_id() + if transaction_id is None: + logger.warning('no index file found for repository %s' % self.path) + else: + index = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8') + logger.info("converting repo index %s" % index) + if not dryrun: + AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX') + + def convert_cache(self, dryrun): + """convert caches from attic to borg + + those are all hash indexes, so we need to + `s/ATTICIDX/BORG_IDX/` in a few locations: * the `files` and `chunks` cache (in `$ATTIC_CACHE_DIR` or `$HOME/.cache/attic//`), which we could just drop, @@ -168,15 +184,6 @@ class AtticRepositoryUpgrader(Repository): `Cache.open()`, edit in place and then `Cache.close()` to make sure we have locking right """ - transaction_id = self.get_index_transaction_id() - if transaction_id is None: - logger.warning('no index file found for repository %s' % self.path) - else: - index = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8') - logger.info("converting index index %s" % index) - if not dryrun: - AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX') - # copy of attic's get_cache_dir() attic_cache_dir = os.environ.get('ATTIC_CACHE_DIR', os.path.join(os.path.expanduser('~'), From 60babd14c3df852870eefeed59776c7b3c40b4f4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 10 Dec 2015 11:59:13 +0100 Subject: [PATCH 152/321] borg upgrade - do not overwrite backup_repo/index.*, fixes #466 the code obviously wrote to both index.* files as they were hardlinked. now we make a normal copy of index (and also hints) to avoid this. --- borg/upgrader.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/borg/upgrader.py b/borg/upgrader.py index 4c14c856..57412b5c 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -37,6 +37,16 @@ class AtticRepositoryUpgrader(Repository): logger.info('making a hardlink copy in %s', backup) if not dryrun: shutil.copytree(self.path, backup, copy_function=os.link) + # we need to create a real copy (not hardlink copy) of index.* and hints.* + # otherwise the backup copy will get modified. + transaction_id = self.get_index_transaction_id() + for name in ['index', 'hints']: + fname = "%s.%d" % (name, transaction_id) + fname_orig = os.path.join(self.path, fname) + fname_backup = os.path.join(backup, fname) + os.remove(fname_backup) + shutil.copy(fname_orig, fname_backup) + logger.info("opening attic repository with borg and converting") # now lock the repo, after we have made the copy self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() From e3d5898addaeb896559594d74d4d17c0340eda89 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 11 Dec 2015 22:18:18 +0100 Subject: [PATCH 153/321] borg upgrade - use inplace parameter, fixes #466 i checked it: copying the index.* and hints.* files in advance is not needed, open() and close() do not modify them. also: fix unicode exception with encoded filename --- borg/upgrader.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index 57412b5c..3bd5400f 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -37,16 +37,6 @@ class AtticRepositoryUpgrader(Repository): logger.info('making a hardlink copy in %s', backup) if not dryrun: shutil.copytree(self.path, backup, copy_function=os.link) - # we need to create a real copy (not hardlink copy) of index.* and hints.* - # otherwise the backup copy will get modified. - transaction_id = self.get_index_transaction_id() - for name in ['index', 'hints']: - fname = "%s.%d" % (name, transaction_id) - fname_orig = os.path.join(self.path, fname) - fname_backup = os.path.join(backup, fname) - os.remove(fname_backup) - shutil.copy(fname_orig, fname_backup) - logger.info("opening attic repository with borg and converting") # now lock the repo, after we have made the copy self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() @@ -62,8 +52,8 @@ class AtticRepositoryUpgrader(Repository): self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire() try: - self.convert_repo_index(dryrun) self.convert_cache(dryrun) + self.convert_repo_index(dryrun=dryrun, inplace=inplace) self.convert_segments(segments, dryrun=dryrun, inplace=inplace) self.borg_readme() finally: @@ -161,7 +151,7 @@ class AtticRepositoryUpgrader(Repository): with open(keyfile, 'w') as f: f.write(data) - def convert_repo_index(self, dryrun): + def convert_repo_index(self, dryrun, inplace): """convert some repo files those are all hash indexes, so we need to @@ -177,10 +167,10 @@ class AtticRepositoryUpgrader(Repository): if transaction_id is None: logger.warning('no index file found for repository %s' % self.path) else: - index = os.path.join(self.path, 'index.%d' % transaction_id).encode('utf-8') + index = os.path.join(self.path, 'index.%d' % transaction_id) logger.info("converting repo index %s" % index) if not dryrun: - AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX') + AtticRepositoryUpgrader.header_replace(index, b'ATTICIDX', b'BORG_IDX', inplace=inplace) def convert_cache(self, dryrun): """convert caches from attic to borg From c4610c1edf4586173e22d1b62f8181bf5ea9d1bc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 22:02:10 +0100 Subject: [PATCH 154/321] remove old mailing list archive references the old archives were merged into the new archives. --- README.rst | 3 +-- docs/support.rst | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/README.rst b/README.rst index c05f34ee..47e2f1c0 100644 --- a/README.rst +++ b/README.rst @@ -121,8 +121,7 @@ Links * `GitHub `_ * `Issue Tracker `_ * `Bounties & Fundraisers `_ - * `New Mailing List `_ - * `(Old Mailing List's Archives `_) + * `Mailing List `_ * `License `_ Notes diff --git a/docs/support.rst b/docs/support.rst index 503fb81e..c2a8e389 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -33,10 +33,6 @@ New Mailing List: Just read that page to find out about the mailing list, its topic, how to subscribe, how to unsubscribe and where you can find the archives of the list. -We used a different mailing list before, you can still read its archives there: - -`Old Mailing List's Archives `_ - Bounties and Fundraisers ------------------------ From 9c271afefa82f68021e46bbdbc211c1a7770a65d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 13:50:24 +0100 Subject: [PATCH 155/321] unify repo/archive parameter name to "location" --- borg/archiver.py | 98 +++++++++++++++++++++---------------------- borg/key.py | 2 +- borg/testsuite/key.py | 2 +- 3 files changed, 51 insertions(+), 51 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 3583870c..0edc180d 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -84,8 +84,8 @@ class Archiver: def do_init(self, args): """Initialize an empty repository""" - logger.info('Initializing repository at "%s"' % args.repository.canonical_path()) - repository = self.open_repository(args.repository, create=True, exclusive=True) + logger.info('Initializing repository at "%s"' % args.location.canonical_path()) + repository = self.open_repository(args.location, create=True, exclusive=True) key = key_creator(repository, args) manifest = Manifest(key, repository) manifest.key = key @@ -96,7 +96,7 @@ class Archiver: def do_check(self, args): """Check repository consistency""" - repository = self.open_repository(args.repository, exclusive=args.repair) + repository = self.open_repository(args.location, exclusive=args.repair) if args.repair: msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + @@ -108,14 +108,14 @@ class Archiver: if not repository.check(repair=args.repair, save_space=args.save_space): return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( - repository, repair=args.repair, archive=args.repository.archive, + repository, repair=args.repair, archive=args.location.archive, last=args.last, save_space=args.save_space): return EXIT_WARNING return EXIT_SUCCESS def do_change_passphrase(self, args): """Change repository key file passphrase""" - repository = self.open_repository(args.repository) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) key.change_passphrase() return EXIT_SUCCESS @@ -126,13 +126,13 @@ class Archiver: dry_run = args.dry_run t0 = datetime.now() if not dry_run: - repository = self.open_repository(args.archive, exclusive=True) + repository = self.open_repository(args.location, exclusive=True) manifest, key = Manifest.load(repository) compr_args = dict(buffer=COMPR_BUFFER) compr_args.update(args.compression) key.compressor = Compressor(**compr_args) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.archive.archive, cache=cache, + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner, progress=args.progress, chunker_params=args.chunker_params, start=t0) @@ -146,9 +146,9 @@ class Archiver: except IOError: pass # Add local repository dir to inode_skip list - if not args.archive.host: + if not args.location.host: try: - st = os.stat(args.archive.path) + st = os.stat(args.location.path) skip_inodes.add((st.st_ino, st.st_dev)) except IOError: pass @@ -271,9 +271,9 @@ class Archiver: logger.warning('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.') if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') - repository = self.open_repository(args.archive) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, args.archive.archive, + archive = Archive(repository, key, manifest, args.location.archive, numeric_owner=args.numeric_owner) patterns = adjust_patterns(args.paths, args.excludes) dry_run = args.dry_run @@ -313,10 +313,10 @@ class Archiver: def do_rename(self, args): """Rename an existing archive""" - repository = self.open_repository(args.archive, exclusive=True) + repository = self.open_repository(args.location, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) archive.rename(args.name) manifest.write() repository.commit() @@ -325,11 +325,11 @@ class Archiver: def do_delete(self, args): """Delete an existing repository or archive""" - repository = self.open_repository(args.target, exclusive=True) + repository = self.open_repository(args.location, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - if args.target.archive: - archive = Archive(repository, key, manifest, args.target.archive, cache=cache) + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() archive.delete(stats) manifest.write() @@ -368,11 +368,11 @@ class Archiver: self.print_error('%s: Mountpoint must be a writable directory' % args.mountpoint) return self.exit_code - repository = self.open_repository(args.src) + repository = self.open_repository(args.location) try: manifest, key = Manifest.load(repository) - if args.src.archive: - archive = Archive(repository, key, manifest, args.src.archive) + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive) else: archive = None operations = FuseOperations(key, repository, manifest, archive) @@ -388,10 +388,10 @@ class Archiver: def do_list(self, args): """List archive or repository contents""" - repository = self.open_repository(args.src) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) - if args.src.archive: - archive = Archive(repository, key, manifest, args.src.archive) + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive) if args.short: for item in archive.iter_items(): print(remove_surrogates(item[b'path'])) @@ -432,10 +432,10 @@ class Archiver: def do_info(self, args): """Show archive details such as disk space used""" - repository = self.open_repository(args.archive) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = archive.calc_stats(cache) print('Name:', archive.name) print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) @@ -451,7 +451,7 @@ class Archiver: def do_prune(self, args): """Prune repository archives according to specified rules""" - repository = self.open_repository(args.repository, exclusive=True) + repository = self.open_repository(args.location, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list @@ -506,7 +506,7 @@ class Archiver: # to be implemented. # XXX: should auto-detect if it is an attic repository here - repo = AtticRepositoryUpgrader(args.repository.path, create=False) + repo = AtticRepositoryUpgrader(args.location.path, create=False) try: repo.upgrade(args.dry_run, inplace=args.inplace) except NotImplementedError as e: @@ -515,9 +515,9 @@ class Archiver: def do_debug_dump_archive_items(self, args): """dump (decrypted, decompressed) archive items metadata (not: data)""" - repository = self.open_repository(args.archive) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, args.archive.archive) + archive = Archive(repository, key, manifest, args.location.archive) for i, item_id in enumerate(archive.metadata[b'items']): data = key.decrypt(item_id, repository.get(item_id)) filename = '%06d_%s.items' %(i, hexlify(item_id).decode('ascii')) @@ -529,7 +529,7 @@ class Archiver: def do_debug_get_obj(self, args): """get object contents from the repository and write it into file""" - repository = self.open_repository(args.repository) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) hex_id = args.id try: @@ -549,7 +549,7 @@ class Archiver: def do_debug_put_obj(self, args): """put file(s) contents into the repository""" - repository = self.open_repository(args.repository) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) for path in args.paths: with open(path, "rb") as f: @@ -562,7 +562,7 @@ class Archiver: def do_debug_delete_obj(self, args): """delete the objects with the given IDs from the repo""" - repository = self.open_repository(args.repository) + repository = self.open_repository(args.location) manifest, key = Manifest.load(repository) modified = False for hex_id in args.ids: @@ -584,7 +584,7 @@ class Archiver: def do_break_lock(self, args): """Break the repository lock (e.g. in case it was left by a dead borg.""" - repository = self.open_repository(args.repository, lock=False) + repository = self.open_repository(args.location, lock=False) try: repository.break_lock() Cache.break_lock(repository) @@ -703,7 +703,7 @@ class Archiver: description=self.do_init.__doc__, epilog=init_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_init) - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', @@ -751,7 +751,7 @@ class Archiver: epilog=check_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_check) - subparser.add_argument('repository', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository or archive to check consistency of') subparser.add_argument('--repository-only', dest='repo_only', action='store_true', @@ -779,7 +779,7 @@ class Archiver: epilog=change_passphrase_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_change_passphrase) - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) create_epilog = textwrap.dedent(""" @@ -853,7 +853,7 @@ class Archiver: subparser.add_argument('-n', '--dry-run', dest='dry_run', action='store_true', default=False, help='do not create a backup archive') - subparser.add_argument('archive', metavar='ARCHIVE', + subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='name of archive to create (must be also a valid directory name)') subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, @@ -893,7 +893,7 @@ class Archiver: subparser.add_argument('--sparse', dest='sparse', action='store_true', default=False, help='create holes in output sparse file from all-zero chunks') - subparser.add_argument('archive', metavar='ARCHIVE', + subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to extract') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, @@ -907,7 +907,7 @@ class Archiver: epilog=rename_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_rename) - subparser.add_argument('archive', metavar='ARCHIVE', + subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to rename') subparser.add_argument('name', metavar='NEWNAME', type=str, @@ -932,7 +932,7 @@ class Archiver: subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('target', metavar='TARGET', nargs='?', default='', + subparser.add_argument('location', metavar='TARGET', nargs='?', default='', type=location_validator(), help='archive or repository to delete') @@ -949,7 +949,7 @@ class Archiver: help='only print file/directory names, nothing else') subparser.add_argument('-p', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') - subparser.add_argument('src', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository/archive to list contents of') @@ -964,7 +964,7 @@ class Archiver: epilog=mount_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_mount) - subparser.add_argument('src', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), + subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='repository/archive to mount') subparser.add_argument('mountpoint', metavar='MOUNTPOINT', type=str, help='where to mount filesystem') @@ -982,7 +982,7 @@ class Archiver: epilog=info_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_info) - subparser.add_argument('archive', metavar='ARCHIVE', + subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to display information about') @@ -996,7 +996,7 @@ class Archiver: epilog=break_lock_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_break_lock) - subparser.add_argument('repository', metavar='REPOSITORY', + subparser.add_argument('location', metavar='REPOSITORY', type=location_validator(archive=False), help='repository for which to break the locks') @@ -1052,7 +1052,7 @@ class Archiver: subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to prune') @@ -1104,7 +1104,7 @@ class Archiver: default=False, action='store_true', help="""rewrite repository in place, with no chance of going back to older versions of the repository.""") - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='path to the repository to be upgraded') @@ -1126,7 +1126,7 @@ class Archiver: epilog=debug_dump_archive_items_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_debug_dump_archive_items) - subparser.add_argument('archive', metavar='ARCHIVE', + subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to dump') @@ -1138,7 +1138,7 @@ class Archiver: epilog=debug_get_obj_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_debug_get_obj) - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to use') subparser.add_argument('id', metavar='ID', type=str, @@ -1154,7 +1154,7 @@ class Archiver: epilog=debug_put_obj_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_debug_put_obj) - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to use') subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, @@ -1168,7 +1168,7 @@ class Archiver: epilog=debug_delete_obj_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_debug_delete_obj) - subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), help='repository to use') subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, diff --git a/borg/key.py b/borg/key.py index a712cd13..a39b1e31 100644 --- a/borg/key.py +++ b/borg/key.py @@ -363,7 +363,7 @@ class KeyfileKey(KeyfileKeyBase): raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir()) def get_new_target(self, args): - filename = args.repository.to_key_filename() + filename = args.location.to_key_filename() path = filename i = 1 while os.path.exists(path): diff --git a/borg/testsuite/key.py b/borg/testsuite/key.py index 2f234dd8..b2011d8f 100644 --- a/borg/testsuite/key.py +++ b/borg/testsuite/key.py @@ -13,7 +13,7 @@ from . import BaseTestCase class KeyTestCase(BaseTestCase): class MockArgs: - repository = Location(tempfile.mkstemp()[1]) + location = Location(tempfile.mkstemp()[1]) keyfile2_key_file = """ BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000 From c194f3ca1c09594fb29004e57a3ce84196684ede Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 13:58:16 +0100 Subject: [PATCH 156/321] give (all) args to open_repository --- borg/archiver.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 0edc180d..65819b66 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -55,7 +55,8 @@ class Archiver: self.exit_code = EXIT_SUCCESS self.lock_wait = lock_wait - def open_repository(self, location, create=False, exclusive=False, lock=True): + def open_repository(self, args, create=False, exclusive=False, lock=True): + location = args.location # note: 'location' must be always present in args if location.proto == 'ssh': repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock) else: @@ -85,7 +86,7 @@ class Archiver: def do_init(self, args): """Initialize an empty repository""" logger.info('Initializing repository at "%s"' % args.location.canonical_path()) - repository = self.open_repository(args.location, create=True, exclusive=True) + repository = self.open_repository(args, create=True, exclusive=True) key = key_creator(repository, args) manifest = Manifest(key, repository) manifest.key = key @@ -96,7 +97,7 @@ class Archiver: def do_check(self, args): """Check repository consistency""" - repository = self.open_repository(args.location, exclusive=args.repair) + repository = self.open_repository(args, exclusive=args.repair) if args.repair: msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + @@ -115,7 +116,7 @@ class Archiver: def do_change_passphrase(self, args): """Change repository key file passphrase""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) key.change_passphrase() return EXIT_SUCCESS @@ -126,7 +127,7 @@ class Archiver: dry_run = args.dry_run t0 = datetime.now() if not dry_run: - repository = self.open_repository(args.location, exclusive=True) + repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) compr_args = dict(buffer=COMPR_BUFFER) compr_args.update(args.compression) @@ -271,7 +272,7 @@ class Archiver: logger.warning('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.') if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.location.archive, numeric_owner=args.numeric_owner) @@ -313,7 +314,7 @@ class Archiver: def do_rename(self, args): """Rename an existing archive""" - repository = self.open_repository(args.location, exclusive=True) + repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, lock_wait=self.lock_wait) archive = Archive(repository, key, manifest, args.location.archive, cache=cache) @@ -325,7 +326,7 @@ class Archiver: def do_delete(self, args): """Delete an existing repository or archive""" - repository = self.open_repository(args.location, exclusive=True) + repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) if args.location.archive: @@ -368,7 +369,7 @@ class Archiver: self.print_error('%s: Mountpoint must be a writable directory' % args.mountpoint) return self.exit_code - repository = self.open_repository(args.location) + repository = self.open_repository(args) try: manifest, key = Manifest.load(repository) if args.location.archive: @@ -388,7 +389,7 @@ class Archiver: def do_list(self, args): """List archive or repository contents""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) if args.location.archive: archive = Archive(repository, key, manifest, args.location.archive) @@ -432,7 +433,7 @@ class Archiver: def do_info(self, args): """Show archive details such as disk space used""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archive = Archive(repository, key, manifest, args.location.archive, cache=cache) @@ -451,7 +452,7 @@ class Archiver: def do_prune(self, args): """Prune repository archives according to specified rules""" - repository = self.open_repository(args.location, exclusive=True) + repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list @@ -515,7 +516,7 @@ class Archiver: def do_debug_dump_archive_items(self, args): """dump (decrypted, decompressed) archive items metadata (not: data)""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.location.archive) for i, item_id in enumerate(archive.metadata[b'items']): @@ -529,7 +530,7 @@ class Archiver: def do_debug_get_obj(self, args): """get object contents from the repository and write it into file""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) hex_id = args.id try: @@ -549,7 +550,7 @@ class Archiver: def do_debug_put_obj(self, args): """put file(s) contents into the repository""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) for path in args.paths: with open(path, "rb") as f: @@ -562,7 +563,7 @@ class Archiver: def do_debug_delete_obj(self, args): """delete the objects with the given IDs from the repo""" - repository = self.open_repository(args.location) + repository = self.open_repository(args) manifest, key = Manifest.load(repository) modified = False for hex_id in args.ids: @@ -584,7 +585,7 @@ class Archiver: def do_break_lock(self, args): """Break the repository lock (e.g. in case it was left by a dead borg.""" - repository = self.open_repository(args.location, lock=False) + repository = self.open_repository(args, lock=False) try: repository.break_lock() Cache.break_lock(repository) From eab60cce993b45c66796de45d7dbf192fb5aa653 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 15:31:43 +0100 Subject: [PATCH 157/321] pass through some global options from client to server new: logging level options refactored: - umask option and remote_path - cleanly separated ssh command from borg command --- borg/archiver.py | 11 +++++----- borg/remote.py | 42 +++++++++++++++++++++--------------- borg/testsuite/repository.py | 26 ++++++++++++++++------ borg/testsuite/upgrader.py | 6 +++--- 4 files changed, 54 insertions(+), 31 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 65819b66..6e143e25 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -34,6 +34,9 @@ from .remote import RepositoryServer, RemoteRepository has_lchflags = hasattr(os, 'lchflags') +# default umask, overriden by --umask, defaults to read/write only for owner +UMASK_DEFAULT = 0o077 + class ToggleAction(argparse.Action): """argparse action to handle "toggle" flags easily @@ -58,7 +61,7 @@ class Archiver: def open_repository(self, args, create=False, exclusive=False, lock=True): location = args.location # note: 'location' must be always present in args if location.proto == 'ssh': - repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock) + repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args) else: repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait, lock=lock) repository._location = location @@ -674,9 +677,9 @@ class Archiver: help='show/log the return code (rc)') common_parser.add_argument('--no-files-cache', dest='cache_files', action='store_false', help='do not load/update the file metadata cache used to detect unchanged files') - common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=RemoteRepository.umask, metavar='M', + common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, metavar='M', help='set umask to M (local and remote, default: %(default)04o)') - common_parser.add_argument('--remote-path', dest='remote_path', default=RemoteRepository.remote_path, metavar='PATH', + common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH', help='set remote path to executable (default: "%(default)s")') parser = argparse.ArgumentParser(prog=prog, description='Borg - Deduplicated Backups') @@ -1188,8 +1191,6 @@ class Archiver: def run(self, args): os.umask(args.umask) # early, before opening files self.lock_wait = args.lock_wait - RemoteRepository.remote_path = args.remote_path - RemoteRepository.umask = args.umask setup_logging(level=args.log_level) # do not use loggers before this! check_extension_modules() keys_dir = get_keys_dir() diff --git a/borg/remote.py b/borg/remote.py index f724b80d..1363d170 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -117,15 +117,12 @@ class RepositoryServer: # pragma: no cover class RemoteRepository: extra_test_args = [] - remote_path = 'borg' - # default umask, overriden by --umask, defaults to read/write only for owner - umask = 0o077 class RPCError(Exception): def __init__(self, name): self.name = name - def __init__(self, location, create=False, lock_wait=None, lock=True): + def __init__(self, location, create=False, lock_wait=None, lock=True, args=None): self.location = location self.preload_ids = [] self.msgid = 0 @@ -135,15 +132,11 @@ class RemoteRepository: self.responses = {} self.unpacker = msgpack.Unpacker(use_list=False) self.p = None - # XXX: ideally, the testsuite would subclass Repository and - # override ssh_cmd() instead of this crude hack, although - # __testsuite__ is not a valid domain name so this is pretty - # safe. - if location.host == '__testsuite__': - args = [sys.executable, '-m', 'borg.archiver', 'serve' ] + self.extra_test_args - else: # pragma: no cover - args = self.ssh_cmd(location) - self.p = Popen(args, bufsize=0, stdin=PIPE, stdout=PIPE) + testing = location.host == '__testsuite__' + borg_cmd = self.borg_cmd(args, testing) + if not testing: + borg_cmd = self.ssh_cmd(location) + borg_cmd + self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) @@ -165,10 +158,27 @@ class RemoteRepository: def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path()) - def umask_flag(self): - return ['--umask', '%03o' % self.umask] + def borg_cmd(self, args, testing): + """return a borg serve command line""" + # give some args/options to "borg serve" process as they were given to us + opts = [] + if args is not None: + opts.append('--umask=%03o' % args.umask) + if args.log_level == 'debug': + opts.append('--debug') + elif args.log_level == 'info': + opts.append('--info') + elif args.log_level == 'warning': + pass # is default + else: + raise ValueError('log level missing, fix this code') + if testing: + return [sys.executable, '-m', 'borg.archiver', 'serve' ] + opts + self.extra_test_args + else: # pragma: no cover + return [args.remote_path, 'serve'] + opts def ssh_cmd(self, location): + """return a ssh command line that can be prefixed to a borg command line""" args = shlex.split(os.environ.get('BORG_RSH', 'ssh')) if location.port: args += ['-p', str(location.port)] @@ -176,8 +186,6 @@ class RemoteRepository: args.append('%s@%s' % (location.user, location.host)) else: args.append('%s' % location.host) - # use local umask also for the remote process - args += [self.remote_path, 'serve'] + self.umask_flag() return args def call(self, cmd, *args, **kw): diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 4094df40..7b2874c5 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -1,5 +1,6 @@ import os import shutil +import sys import tempfile from mock import patch @@ -326,13 +327,26 @@ class RemoteRepositoryTestCase(RepositoryTestCase): self.assert_raises(InvalidRPCMethod, lambda: self.repository.call('__init__', None)) def test_ssh_cmd(self): - assert self.repository.umask is not None - assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() - assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() - assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag() - assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com', 'borg', 'serve'] + self.repository.umask_flag() + assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', 'example.com'] + assert self.repository.ssh_cmd(Location('ssh://example.com/foo')) == ['ssh', 'example.com'] + assert self.repository.ssh_cmd(Location('ssh://user@example.com/foo')) == ['ssh', 'user@example.com'] + assert self.repository.ssh_cmd(Location('ssh://user@example.com:1234/foo')) == ['ssh', '-p', '1234', 'user@example.com'] os.environ['BORG_RSH'] = 'ssh --foo' - assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com', 'borg', 'serve'] + self.repository.umask_flag() + assert self.repository.ssh_cmd(Location('example.com:foo')) == ['ssh', '--foo', 'example.com'] + + def test_borg_cmd(self): + class MockArgs: + remote_path = 'borg' + log_level = 'warning' + umask = 0o077 + + assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve' ] + args = MockArgs() + assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077'] + args.log_level = 'info' + assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info'] + args.remote_path = 'borg-0.28.2' + assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--umask=077', '--info'] class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase): diff --git a/borg/testsuite/upgrader.py b/borg/testsuite/upgrader.py index 3d045912..9a1f823f 100644 --- a/borg/testsuite/upgrader.py +++ b/borg/testsuite/upgrader.py @@ -12,7 +12,7 @@ except ImportError: from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey from ..helpers import get_keys_dir from ..key import KeyfileKey -from ..remote import RemoteRepository +from ..archiver import UMASK_DEFAULT from ..repository import Repository @@ -169,7 +169,7 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): orig_inode = first_inode(attic_repo.path) repo = AtticRepositoryUpgrader(str(tmpdir), create=False) # replicate command dispatch, partly - os.umask(RemoteRepository.umask) + os.umask(UMASK_DEFAULT) backup = repo.upgrade(dryrun=False, inplace=inplace) if inplace: assert backup is None @@ -179,7 +179,7 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): assert first_inode(repo.path) != first_inode(backup) # i have seen cases where the copied tree has world-readable # permissions, which is wrong - assert stat_segment(backup).st_mode & 0o007 == 0 + assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 assert key_valid(attic_key_file.path) assert repo_valid(tmpdir) From f59db03c60a2f6c0d832ad25903037543a4531ab Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 17:24:40 +0100 Subject: [PATCH 158/321] ProgressIndicator: flush the output file or it won't work correctly via ssh likely due to buffering, the progress indication was not visible. --- borg/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index df6f1163..925dfb11 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -912,7 +912,8 @@ class ProgressIndicatorPercent: return self.output(pct) def output(self, percent): - print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n') + print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n') # python 3.3 gives us flush=True + self.file.flush() def finish(self): if self.same_line: From 229512b6f503584e2ae760b2b77f8f81e31a76e7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 18:25:38 +0100 Subject: [PATCH 159/321] determine log level from the logger, so it works with logging.conf also --- borg/remote.py | 10 ++++++---- borg/testsuite/repository.py | 4 +--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index 1363d170..1c02c08e 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -1,5 +1,6 @@ import errno import fcntl +import logging import os import select import shlex @@ -164,12 +165,13 @@ class RemoteRepository: opts = [] if args is not None: opts.append('--umask=%03o' % args.umask) - if args.log_level == 'debug': + root_logger = logging.getLogger() + if root_logger.isEnabledFor(logging.DEBUG): opts.append('--debug') - elif args.log_level == 'info': + elif root_logger.isEnabledFor(logging.INFO): opts.append('--info') - elif args.log_level == 'warning': - pass # is default + elif root_logger.isEnabledFor(logging.WARNING): + pass # warning is default else: raise ValueError('log level missing, fix this code') if testing: diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 7b2874c5..7027cb59 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -337,13 +337,11 @@ class RemoteRepositoryTestCase(RepositoryTestCase): def test_borg_cmd(self): class MockArgs: remote_path = 'borg' - log_level = 'warning' umask = 0o077 assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve' ] args = MockArgs() - assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077'] - args.log_level = 'info' + # note: test logger is on info log level, so --info gets added automagically assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info'] args.remote_path = 'borg-0.28.2' assert self.repository.borg_cmd(args, testing=False) == ['borg-0.28.2', 'serve', '--umask=077', '--info'] From 2e2e145372da9a8877de872fe3ffbb408eb0217a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 21:24:21 +0100 Subject: [PATCH 160/321] sane remote logging, remote stderr, fixes #461 --- borg/archiver.py | 2 +- borg/logger.py | 13 ++++++++---- borg/remote.py | 52 +++++++++++++++++++++++++++++++++--------------- 3 files changed, 46 insertions(+), 21 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6e143e25..a1c580f2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1191,7 +1191,7 @@ class Archiver: def run(self, args): os.umask(args.umask) # early, before opening files self.lock_wait = args.lock_wait - setup_logging(level=args.log_level) # do not use loggers before this! + setup_logging(level=args.log_level, is_serve=args.func == self.do_serve) # do not use loggers before this! check_extension_modules() keys_dir = get_keys_dir() if not os.path.exists(keys_dir): diff --git a/borg/logger.py b/borg/logger.py index a40f676c..f2350f8d 100644 --- a/borg/logger.py +++ b/borg/logger.py @@ -52,7 +52,7 @@ def _log_warning(message, category, filename, lineno, file=None, line=None): logger.warning(msg) -def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info'): +def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', level='info', is_serve=False): """setup logging module according to the arguments provided if conf_fname is given (or the config file name can be determined via @@ -60,6 +60,9 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev otherwise, set up a stream handler logger on stderr (by default, if no stream is provided). + + if is_serve == True, we configure a special log format as expected by + the borg client log message interceptor. """ global configured err_msg = None @@ -84,9 +87,11 @@ def setup_logging(stream=None, conf_fname=None, env_var='BORG_LOGGING_CONF', lev # if we did not / not successfully load a logging configuration, fallback to this: logger = logging.getLogger('') handler = logging.StreamHandler(stream) - # other formatters will probably want this, but let's remove clutter on stderr - # example: - # handler.setFormatter(logging.Formatter('%(name)s: %(message)s')) + if is_serve: + fmt = '$LOG %(levelname)s Remote: %(message)s' + else: + fmt = '%(message)s' + handler.setFormatter(logging.Formatter(fmt)) logger.addHandler(handler) logger.setLevel(level.upper()) configured = True diff --git a/borg/remote.py b/borg/remote.py index 1c02c08e..2f6c58ff 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -63,12 +63,16 @@ class RepositoryServer: # pragma: no cover def serve(self): stdin_fd = sys.stdin.fileno() stdout_fd = sys.stdout.fileno() + stderr_fd = sys.stdout.fileno() # Make stdin non-blocking fl = fcntl.fcntl(stdin_fd, fcntl.F_GETFL) fcntl.fcntl(stdin_fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) # Make stdout blocking fl = fcntl.fcntl(stdout_fd, fcntl.F_GETFL) fcntl.fcntl(stdout_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK) + # Make stderr blocking + fl = fcntl.fcntl(stderr_fd, fcntl.F_GETFL) + fcntl.fcntl(stderr_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK) unpacker = msgpack.Unpacker(use_list=False) while True: r, w, es = select.select([stdin_fd], [], [], 10) @@ -91,6 +95,7 @@ class RepositoryServer: # pragma: no cover f = getattr(self.repository, method) res = f(*args) except BaseException as e: + # XXX rather log exception exc = "Remote Traceback by Borg %s%s%s" % (__version__, os.linesep, traceback.format_exc()) os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: @@ -137,13 +142,15 @@ class RemoteRepository: borg_cmd = self.borg_cmd(args, testing) if not testing: borg_cmd = self.ssh_cmd(location) + borg_cmd - self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE) + self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() + self.stderr_fd = self.p.stderr.fileno() fcntl.fcntl(self.stdin_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdin_fd, fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(self.stdout_fd, fcntl.F_SETFL, fcntl.fcntl(self.stdout_fd, fcntl.F_GETFL) | os.O_NONBLOCK) - self.r_fds = [self.stdout_fd] - self.x_fds = [self.stdin_fd, self.stdout_fd] + fcntl.fcntl(self.stderr_fd, fcntl.F_SETFL, fcntl.fcntl(self.stderr_fd, fcntl.F_GETFL) | os.O_NONBLOCK) + self.r_fds = [self.stdout_fd, self.stderr_fd] + self.x_fds = [self.stdin_fd, self.stdout_fd, self.stderr_fd] try: version = self.call('negotiate', RPC_PROTOCOL_VERSION) @@ -238,19 +245,32 @@ class RemoteRepository: r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) if x: raise Exception('FD exception occurred') - if r: - data = os.read(self.stdout_fd, BUFSIZE) - if not data: - raise ConnectionClosed() - self.unpacker.feed(data) - for unpacked in self.unpacker: - if not (isinstance(unpacked, tuple) and len(unpacked) == 4): - raise Exception("Unexpected RPC data format.") - type, msgid, error, res = unpacked - if msgid in self.ignore_responses: - self.ignore_responses.remove(msgid) - else: - self.responses[msgid] = error, res + for fd in r: + if fd is self.stdout_fd: + data = os.read(fd, BUFSIZE) + if not data: + raise ConnectionClosed() + self.unpacker.feed(data) + for unpacked in self.unpacker: + if not (isinstance(unpacked, tuple) and len(unpacked) == 4): + raise Exception("Unexpected RPC data format.") + type, msgid, error, res = unpacked + if msgid in self.ignore_responses: + self.ignore_responses.remove(msgid) + else: + self.responses[msgid] = error, res + elif fd is self.stderr_fd: + data = os.read(fd, 32768) + if not data: + raise ConnectionClosed() + data = data.decode('utf-8') + for line in data.splitlines(): + if line.startswith('$LOG '): + _, level, msg = line.split(' ', 2) + level = getattr(logging, level, logging.CRITICAL) # str -> int + logging.log(level, msg) + else: + print("Remote: " + line, file=sys.stderr) if w: while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < 100: if calls: From 2df0bb1f8344b4fc2118b7a1a597e69e4583f0fa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 22:13:41 +0100 Subject: [PATCH 161/321] remote stderr: keep line endings as is so even the \r trick works for overwriting the same line. --- borg/remote.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index 2f6c58ff..64745f64 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -264,13 +264,13 @@ class RemoteRepository: if not data: raise ConnectionClosed() data = data.decode('utf-8') - for line in data.splitlines(): + for line in data.splitlines(keepends=True): if line.startswith('$LOG '): _, level, msg = line.split(' ', 2) level = getattr(logging, level, logging.CRITICAL) # str -> int - logging.log(level, msg) + logging.log(level, msg.rstrip()) else: - print("Remote: " + line, file=sys.stderr) + sys.stderr.write("Remote: " + line) if w: while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < 100: if calls: From 942120997e674d940879791801ad1ade82d2a1c0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Dec 2015 22:45:29 +0100 Subject: [PATCH 162/321] log remote exceptions, add remote sysinfo --- borg/archiver.py | 2 +- borg/remote.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index a1c580f2..ff0c259b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1263,7 +1263,7 @@ def main(): # pragma: no cover msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) exit_code = e.exit_code except RemoteRepository.RPCError as e: - msg = 'Remote Exception.\n%s\n%s' % (str(e), sysinfo()) + msg = '%s\n%s' % (str(e), sysinfo()) exit_code = EXIT_ERROR except Exception: msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo()) diff --git a/borg/remote.py b/borg/remote.py index 64745f64..50b56e1c 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -11,7 +11,7 @@ import traceback from . import __version__ -from .helpers import Error, IntegrityError +from .helpers import Error, IntegrityError, sysinfo from .repository import Repository import msgpack @@ -95,8 +95,9 @@ class RepositoryServer: # pragma: no cover f = getattr(self.repository, method) res = f(*args) except BaseException as e: - # XXX rather log exception - exc = "Remote Traceback by Borg %s%s%s" % (__version__, os.linesep, traceback.format_exc()) + logging.exception('Borg %s: exception in RPC call:', __version__) + logging.error(sysinfo()) + exc = "Remote Exception (see remote log for the traceback)" os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: os.write(stdout_fd, msgpack.packb((1, msgid, None, res))) From aa97724c0cb07cf712f0ae38f6826cd55568c164 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 00:39:15 +0100 Subject: [PATCH 163/321] add --prefix to check to check only some specific archives, fixes #206 --- borg/archive.py | 12 +++++++----- borg/archiver.py | 4 +++- borg/testsuite/archiver.py | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 011d7845..390c64ff 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -661,9 +661,9 @@ class ArchiveChecker: self.error_found = False self.possibly_superseded = set() - def check(self, repository, repair=False, archive=None, last=None, save_space=False): + def check(self, repository, repair=False, archive=None, last=None, prefix=None, save_space=False): logger.info('Starting archive consistency check...') - self.check_all = archive is None and last is None + self.check_all = archive is None and last is None and prefix is None self.repair = repair self.repository = repository self.init_chunks() @@ -674,7 +674,7 @@ class ArchiveChecker: self.manifest = self.rebuild_manifest() else: self.manifest, _ = Manifest.load(repository, key=self.key) - self.rebuild_refcounts(archive=archive, last=last) + self.rebuild_refcounts(archive=archive, last=last, prefix=prefix) self.orphan_chunks_check() self.finish(save_space=save_space) if self.error_found: @@ -730,7 +730,7 @@ class ArchiveChecker: logger.info('Manifest rebuild complete.') return manifest - def rebuild_refcounts(self, archive=None, last=None): + def rebuild_refcounts(self, archive=None, last=None, prefix=None): """Rebuild object reference counts by walking the metadata Missing and/or incorrect data is repaired when detected @@ -830,7 +830,9 @@ class ArchiveChecker: # we need last N or all archives archive_items = sorted(self.manifest.archives.items(), reverse=True, key=lambda name_info: name_info[1][b'time']) - num_archives = len(self.manifest.archives) + if prefix is not None: + archive_items = [item for item in archive_items if item[0].startswith(prefix)] + num_archives = len(archive_items) end = None if last is None else min(num_archives, last) else: # we only want one specific archive diff --git a/borg/archiver.py b/borg/archiver.py index 3583870c..e566c572 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -109,7 +109,7 @@ class Archiver: return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( repository, repair=args.repair, archive=args.repository.archive, - last=args.last, save_space=args.save_space): + last=args.last, prefix=args.prefix, save_space=args.save_space): return EXIT_WARNING return EXIT_SUCCESS @@ -769,6 +769,8 @@ class Archiver: subparser.add_argument('--last', dest='last', type=int, default=None, metavar='N', help='only check last N archives (Default: all)') + subparser.add_argument('-p', '--prefix', dest='prefix', type=str, + help='only consider archive names starting with this prefix') change_passphrase_epilog = textwrap.dedent(""" The key files used for repository encryption are optionally passphrase diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d8ae92cc..c20a5182 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -949,6 +949,8 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0) self.assert_not_in('Starting repository check', output) self.assert_in('Starting archive consistency check', output) + output = self.cmd('check', '-v', '--archives-only', '--prefix=archive2', self.repository_location, exit_code=0) + self.assert_not_in('archive1', output) def test_missing_file_chunk(self): archive, repository = self.open_archive('archive1') From 0c166898bfe03fed3f73cd3ecf9ca478fc614345 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 00:51:39 +0100 Subject: [PATCH 164/321] fix python 3.2 str.splitlines() compatibility issue --- borg/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/remote.py b/borg/remote.py index 50b56e1c..1fcd97c5 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -265,7 +265,7 @@ class RemoteRepository: if not data: raise ConnectionClosed() data = data.decode('utf-8') - for line in data.splitlines(keepends=True): + for line in data.splitlines(True): # keepends=True for py3.3+ if line.startswith('$LOG '): _, level, msg = line.split(' ', 2) level = getattr(logging, level, logging.CRITICAL) # str -> int From f3d60fdb377045971d5cf5f7ae2d0379cdb0e76c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 15:38:23 +0100 Subject: [PATCH 165/321] update CHANGES --- docs/changes.rst | 62 +++++++++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index aa7dc7e6..5c6e8121 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,14 +1,18 @@ Changelog ========= -Version 0.29.0 (not released yet) ---------------------------------- +Version 0.29.0 +-------------- Compatibility notes: - when upgrading to 0.29.0 you need to upgrade client as well as server - installations due to the locking related changes otherwise you'll get an - error msg about a RPC protocol mismatch. + installations due to the locking and commandline interface changes otherwise + you'll get an error msg about a RPC protocol mismatch or a wrong commandline + option. + if you run a server that needs to support both old and new clients, it is + suggested that you have a "borg-0.28.2" and a "borg-0.29.0" command. + clients then can choose via e.g. "borg --remote-path=borg-0.29.0 ...". - the default waiting time for a lock changed from infinity to 1 second for a better interactive user experience. if the repo you want to access is currently locked, borg will now terminate after 1s with an error message. @@ -19,34 +23,43 @@ Bug fixes: - hash table tuning (better chosen hashtable load factor 0.75 and prime initial size of 1031 gave ~1000x speedup in some scenarios) -- avoid creation of an orphan lock for one case, see #285 +- avoid creation of an orphan lock for one case, #285 - --keep-tag-files: fix file mode and multiple tag files in one directory, #432 -- fix format of umask in help pages, #463 +- fixes for "borg upgrade" (attic repo converter), #466 +- remove --progress isatty magic (and also --no-progress option) again, #476 - borg init: display proper repo URL +- fix format of umask in help pages, #463 New features: -- implement --lock-wait, support timeout for UpgradableLock, fixes #210 -- implement borg break-lock command, fixes #157 -- include system info below traceback, fixes #324 -- use ISO-8601 date and time format, fixes #375 +- implement --lock-wait, support timeout for UpgradableLock, #210 +- implement borg break-lock command, #157 +- include system info below traceback, #324 +- sane remote logging, remote stderr, #461: + + - remote log output: intercept it and log it via local logging system, + with "Remote: " prefixed to message. log remote tracebacks. + - remote stderr: output it to local stderr with "Remote: " prefixed. - add --debug and --info (same as --verbose) to set the log level of the - builtin logging configuration (which otherwise defaults to warning), - fixes #426 - note: there are no messages emitted at DEBUG level currently. -- configure logging via env var BORG_LOGGING_CONF -- add a --no-progress flag to forcibly disable progress info -- add a --filter option for status characters: e.g. to show only the added + builtin logging configuration (which otherwise defaults to warning), #426 + note: there are few messages emitted at DEBUG level currently. +- optionally configure logging via env var BORG_LOGGING_CONF +- add --filter option for status characters: e.g. to show only the added or modified files (and also errors), use "borg create -v --filter=AME ...". -- more progress indicators, fixes #394 +- more progress indicators, #394 +- use ISO-8601 date and time format, #375 +- "borg check --prefix" to restrict archive checking to that name prefix, #206 Other changes: -- hashindex_add C implementation (speed up cache resync for new archives) -- increase rpc protocol version to 2, fixes #458 -- silence borg by default (via log level WARNING) -- get rid of C compiler warnings, fixes #391 +- hashindex_add C implementation (speed up cache re-sync for new archives) +- increase FUSE read_size to 1024 (speed up metadata operations) +- check/delete/prune --save-space: free unused segments quickly, #239 +- increase rpc protocol version to 2 (see also Compatibility notes), #458 +- silence borg by default (via default log level WARNING) +- get rid of C compiler warnings, #391 - upgrade OS X FUSE to 3.0.9 on the OS X binary build system +- use python 3.5.1 to build binaries - docs: - new mailing list borgbackup@python.org, #468 @@ -54,13 +67,14 @@ Other changes: - load coverage icons over SSL (avoids mixed content) - more precise binary installation steps - update release procedure docs about OS X FUSE - - FAQ entry about unexpected 'A' status for unchanged file(s), fixes #403 + - FAQ entry about unexpected 'A' status for unchanged file(s), #403 - add docs about 'E' file status - - add "borg upgrade" docs, fixes #464 + - add "borg upgrade" docs, #464 - add developer docs about output and logging - - clarify encryption, add note about clientside encryption + - clarify encryption, add note about client-side encryption - add resources section, with videos, talks, presentations, #149 - Borg moved to Arch Linux [community] + - fix wrong installation instructions for archlinux Version 0.28.2 From 2ecfa54aee99c36d810c5636a1e6f2c48acb9929 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 15:47:03 +0100 Subject: [PATCH 166/321] ran build_api and build_usage --- docs/_static/logo.xcf | Bin 0 -> 26715 bytes docs/api.rst | 48 +++++++++++++++++++------------------- docs/usage/check.rst.inc | 6 ++++- docs/usage/create.rst.inc | 5 ++-- docs/usage/delete.rst.inc | 3 ++- docs/usage/prune.rst.inc | 2 ++ 6 files changed, 35 insertions(+), 29 deletions(-) create mode 100644 docs/_static/logo.xcf diff --git a/docs/_static/logo.xcf b/docs/_static/logo.xcf new file mode 100644 index 0000000000000000000000000000000000000000..ea9c55c26a2e3dbfb03f7356e58726ada9335971 GIT binary patch literal 26715 zcmeHQcUTlxyPuhD>@HnURFqy6v0+2(y~J3sMZFp;5>cwXAjXo<9!rd05{+VyJ&6sa zR~4zE!1kr>E(lU~>3iQZTUcB$xw+4M?)S&;^Bnf<%=yiEXXZWcd8_*QZw#{EwtAiY zI{$#R7<}Z=4x$cxETG{y{OJPCXu?ShLk&Jw(5TQ@(7LMoXq+{Sw}o$WKe()?|HhSm zYd!r!{MW!C49?@2duZ79fVK9afdT$&`mo&B`K=BN2n?}zC%-&H{8p{BAJ~5&`RSlD zNPVggKv#7D)ttBQyn~?ILW5SW_TS`ZKS(nd9NgZ~LG>}u&71tg?EN>bUcWXJ;RDyL z3tbzgB@m(Y52=o{hoa}8{@_P2Y{XLx8!6Eofp7Hi70}SLG@8CTaFe~$An0OrXy|z> zp&6vuu_7`V0Rs`WZ$(ketA`mTv6Z9*woz8(yFvlnqxIM%i76nrvLH#nH4Qc2PGR8 z34zrRJx46ve>FN&P|?sz9<7Aiz>T{5tU4CMtE{bWZdE!eO)xoO<}!KXndik#N;jnu z36&cUUH;v}Y^kD^Jk*K<11Q+;D^5lU>spk3lq^g^G51(-{5ii_>7k^Nn0oWE8-7R- zE0q1<0Ewt)_Y1g~TGpWSR5CCH!FFA6E>+sB#Fcn!UnLz=;ATUXA4@1}Rt|t)^+qns zPeh3ulmnGiI7Qe@I`oXMP!3X3fIw#VkpT~c@RZQcdaCuXuTN*!DZSuFt&!7LN1jPq zl*-mtxDEwtV0-O*4pa<5{jM{vaN)5JQ7SN;%I@bEU8x)feX8YvMMpDh$v(qw&Yrvc zda}>#yD>7oqM7WoybdSJ>RX{-kC}|S#Dj-5T-k_q_x+<>>8)(+J@lKnD%2S@H~e)C z>MZs@l`BV`;Jud%6{z#wnFoS;)cNIlba4af+^4S{th+ysY?#7mtK3 z9p{K+Ju#+I+S*tn6XvBoyLoP3z#nS9ZLY7a@$)kqC%c#tm@)d5 zN2Q{f|NQs@4|58LBqwFPLY(l!hOy=(f`&OO>l?~aF9!5e1E_Efyp;fuH1*s9S2h9i z%@ux?B^+Bm#0VF(Q74wW7{L#Br9#1fc4(3ffnyZ-C2wsKJ_s1uok?H>)>m0y zmwo!{Vb)am;R=_XGgwrGJ*ph$_9n_`93eUs^8kky)vUK^gWe|%4olSY}x5{@p!u>=^UlANFElu)6A7n|YVUnP9L08q5Fi z?M`;8;=UQ$8y%^69X@vy>d13X`7cHtMd9VW!KhO&cyQ)B)M+S=zWxj9G)iASzK1$Z zAM%o8QKz{^keh)zE%H)6hwRi>N=i|uwMkxGMRtHO>S5)=-L`~NN2oZGl7=c>r^dN6>A>Zlf;^S+HOkypwj6bUZ9>GcR%Hp7mEZG0WSO~67o5TKg{x8 zbhEIsL?mDd2%%gc%zgf2$aElyfu`vD<*r8s5)qF};o`iHd>$u1?#wLjh1cJg7YTR_ z9+g|m6PAi{p8gm(ss|moR0a7nzf8#EQn{5pKCdu8?lhXB;6sUskLKmKqEod3v51ou zb$-JnCnm74W-1j<1~{1M^cUWXZsdO`7V`Lffq;+mahR}`2c3FB?(=iO(`@ySRT|`Q z-%DA=yn^@C_f7eQ91fSpm6Mb{Ik(BDQ`F2Mvwlp@&&ha0ylF_!$a+_x!jYst`|%q+ zTrrxhAnw%k{u2(|iFx+q(c?#t{(Kyrmi@jECL}3s668jm(F2N9!0)HH4P3MT`{T!s z9y@yYyJP>pANS@RmyaG#15a2c%6zB?{D^>saVIA9uo^UR=8Wn1^y+Do$Io1KF!Ff@ zM+i^~f)h+T>9bM{C8l6U=)lez)9`K0UUW$NMsQ-u&195?Y##_;C z99>H+oo!u7(@7lIy^X9OSz%2(aJWR~tkZ-kt zCoUJI{-g&a7iB@*>6up`edC5@%UBDez@SHTnT^0S{Dou!BR0JliWDW9eHiyA+cOQkYg)*zLD*h9Q0=Lw1n z6EE)Y?W2d(N)PWvw>VX$A|XxKEFfo-^F)$@KekMDG1mhjA*U8rDmf}7gt(xY&qGwo z`C@7QgP^g!&_3N>$f+kgGs|bmJ(0XZQi_-2WeuezVzi7AnZnY7=SNly?|jBtBfq-M ztCkjvizr2nVj-dw;rL?Ct9wV5yLLPyh^v`HzBn6~lb)JFq{x#~UuEWkxIzLz!54CK zQ+|t>+6glL`>F0;{{K36`s4}xMD6h-Cx5;d{VER$ECOB!k^zZVw&-D1A}jvnMBDB@ za~3aLfG?<-Kkv(h8;)L&&IFMK^Afx!L0){M9#X^O0F`;`kVqyt8jFi+BZ{N*3)L zpUTwE5o)<@GkbSCVbp=7*F0`?_XLGXsUdJ$&$+s?2$(}(7HME(Epbz88;VVJH!JfV z!`2+Vqk2o|^`adqL+@#WLO||<6K!oq%$z@O-rRX}zMSFfe?-Mm>P@gj{4PNcNit(W z95C|W_5V6|<`jOa=KGV!Phb2!Aw#|MHp03SzPYEiX;hjcco$k3*l4V&JA;B_BeAB5 zE@o701lE9K8>jSQfU?(w6TmM<^adJ*Ks_eCnFd3dU`R9_Tj9oFRmDD4aCv0Mx&@h-ev%QOY1s#@u52}xzft=4q#H5w5;^~i?bUi zb*GZBCkMPup`)N!4*6=|gG_;t0uu6rsN_T8?G@wa9DkBkC@c~Q1wvd{C*o!&T-h?r z0?4ESnUD7ak_N5=bI-+lue}mU#XJs38v$Qjp8p$Qk3GtG&qIQb%L5gW$4UPAzXn+W zu@oTo@%|B}aFJ@N^}vOPZl~mnON3kw7w1-Tc;bqJ+h0$Z3p3~QK~n|gmJgcm+o;nU zCIZO_Op;8&YABXoiz4o4fC5TN*D8Kdc>yB&*yGH6o(jsVwSa|>1JzFt z0@XNHjT080i^8u0*&-mD$_0@xD$BXHZ0xM>?x*Dz^0;&^hlj{c`f1rfD<%b;KQV{l zy?a+v_F0boHf|i%En{VtYw>jh%Hd5i(;T!T$*)pb`O|vyggV} z7|UbhFVWdNG5L&3`66!SiwmJXHVhh#&a&v)XKL8Pd}M^sxz&8JEbrPvdtgULcwRUr zXAjtXJvo;L8wr(L#}kM-ZyujnWe+YOjKJkovjMAb=a-8Sw3c7=q43rcCn`oWg;%Tw za!Er40l9A;pI&K41&)#9ru|pl&WERSm$blV07!D zes=)ISHUXsZ!C6%*^p5#XL$xkCbWm0^K)mysXBsbGu;2K1_wIS`|FDxDWAjjQQ=rN zBS;)cW*d&x>3nLOW$b`W_e+7)}H1fWh625U1h{ zHHWK!MH&t#HxrE2AFV{)1@+|2YDXGc8HB~4wPRG+ zPtLA!W`Gx}7B+z{Uh6b#s+eC?0Z6o_9)=Ht*TL+&;pdnfJ~CgaTo4zMM!L1U1$elLnY{poR_wfRzhx0!mR@fy?UA(oAq!TI%&m$X9cCC zXaK86ATCo&tHb~01`!3i?bP6(k_DttSBbiJub=(8d-*u$?smgwEd3_pLLzCkBA*E^ zQ1C!dPDX;Vy-x8Xat*p5%>jH;CZI1+DL8yc9~Dw1-~fSHV)b-WYgW-!jU)s}Qa2>tZjNsD`5?`k#CsC55$th_`&+naD3$Gbt_g!=1x2U9-Daq92 zhWq=+TEkXsg4I&HI4(MHH(OE-dI3dRCM&O$@-uF4>kpnKr!NoxnI({Z09`;v$m;WM zt{sZB88&R&mi-+UA5^2LQd#*21lr=kWDi`l_s%O`i4-un6qhN=`SCvlOt7JX>Qjxh z3y1l8ufOD%fmK5hR0>7K!o1`w!F^17FFSHCH6JW2il9y?67#ZSE(Q%k%Fjnqw{Y;? zb46tUQ()hNeIz^na>#(5L-*XzC=ek>0)-1or-09WAA82vl}!Pq2F6ZYhMDquMRMqi2Wo#}t~eHp(H+)ikdt%XfVke70E z_qcAzC0zl8s+0*`tW1U7J9pZe>lysKoVSHw7Xw(p6AHNb3BRlv(Vc!Km zEaHjE_^FZGXIj&GF8-HVm(ar|M1~GB#!R|&G3hxj{C3(~mAnyw(~FyWJ$$w`-EsZT zF)CG}w+_i@-kgNUU=L8q2?Ard21ROiaS=~Y#7VumZ(di1b1;%bu(xw{vGO)7?Bq0; zKoY7VSs*S{L0u3O5z`^m6K%?M%zmL)VGwd)P~djav^OH=S~HwgSL|e36&CcI8>xAM zB2c_2e33v@!h3lONTxe&ysWyB&gYf02W`KT&MA?WmQYHB#U+x8qRhJo=XYcLE$qN; zw_fIpON+!5F|4=}sqppf{k~oQ7MA6);rye7*BR-rsjo%p89A?>{Iq3;CEa<`Uz2R) z;JfAc`Ad%3&X0eM|Oe2UAx2Yzb%|^pugTrL({(p7xo_~8#$X`4hxKf+i z+ZkQtAdLp_*+H7^o~ygKHH!?m9bL&oyR&ybNXO*pbP#NG(=?m4cK=1ONZLcS(N3V9 z7WQ6B3)t;!dJ|aiIOKCwsH}B&R|nf}R+e3Cz;cJp7I$j^%iWSigAwJ(Lf;+{u{~hX z^s(NfzwkrWI(3H}S?gc>*qfl;TZSz1#H`Gud&jq|Tef=F_4stK&M8TCDK9hQfAt@0 z53xN9Oh%gE@;XW8&7<2kZa$S`b_NHRfehQllRAWr;#-B>#&jFW&BD&X;p3KA~v5Ah2=7R`~sR!Z)r z9IjAOocH4A(DC2}F~MZWN(Ql309Oq03p)0KQwF*j&O@;f0n&tCMy~gn;C^13wCW$Ve*xIE4|0<*S8Y&k z!PipWYR%eWPSV9~v%t4k^B3)|dVSwQM{+28;MSX|Z$Xbx+0`)g=gl*Fsfbk(@VTL3hN1s? zPe8>39lY32-&VwYdle51(BpyjLsPDW&+Vypi*!B|7ELh+hR*8+ZW$1+tj}B)Eig{n zZ{MvLU_m!AF~Zn`w|&ZnS`tc8s3ZN+`K_~uc?=%lWMVaZ$IX=7b}XwNONB)a`d|E= zGh0?KTfJ1EUtBGenDp19x)q$*2(+5mE^dLSu>KXB@ z{z(*MN9C*Iy#&8G`^iZ?ud^nuG;_$8=Ti!Evoa_djhQ*x@N$u;Fyry*HJ!q~7(V}U zb}7FwpPJuVz`;3m6dWoMW<5Id&2T-CI{JlT=P2eb1kWV_L3efNoZvTPKRdH&a(69G z6RStK!uMp%T^9$^l;Bo#xB^7vle6o_+UrKLBw(~tP?%pGach?uzz0Wq)}yl9{!+|bCT;vR=5@RI6E<& z>KuD_GiI9hK);Ng>YW|!x?6(fX`-_HX(qG=Drg)KG_auI)D{YXg&QYpQu;_Ptni-* z=@{S%88m;-oi|{dksBX4XY+GX9{sw1)o|DTb2o$^yNIG<=yl-mI)tQWcPO}$8Dktw z!NUr~51haMR&st(De{6-xU$0c*{>6R`*!-^F#*RS?>`4i3a(2avsoY%zkPPxZ#?N! z?n;9di<0-u`z_jY^_5WV;4SAA7IFlyE-oH5c;3b;K0r-$q4t(VM zhiOsd0N6MAr)2|>YlXzIFd>k;7&-odPlZCbZb>EQ4jHTfY9UxbflvjaG>mlmQ&D_~ zjlBi00J!Z_xzehNa+$3BV?!|+PKhljl7V+bRxXp2zK=Tf%>*(o0`wVUMd-Qiy`E7D zacYS~R;dvG4gr|c(eFhPnFN>A$|S{t)L$T|0i@BZ*|Kz__7dTXzJ1HOR5g^c4ZY zVVlEnVB4_ZP0J^ep`=zSj8+gvrq;cjfLjTtT4xu>o@OlLuDu)_ooG(DV^c2_U16ek zVy0$bBVb_QUka7Rq&0&c4QFwDG#U|Z40^UIH zke5Wwr@F2go$K_*Dm9#_drgaN^z|ZG`DX26Ps~+}F&Q1&`g|m}`T@OnJ5_Ds0%p1awRCc$%s6!Z8^_@m`*u6%sRdsAfZ6e1oEC;P6K{UI$ z*eI-}%P91h>4^SP*bpa0@17ac_U^&nBKHx{w7Ge&qHmu&eCYW8_1^s_hr=e^(S9vP zG71?31-a>oPaZyecKh($(X;m51W$*qLaN@fn^iF<8K*EmFFQMrmwsi{)Oq`FLb#=a z!A)xUNy@pRvMTPqbyMej+sWVtwYbEQSQLs&WYxSo%SKKPztJ%v4O%f%0mH#r^5*9m zeY|(v==dfZ)Y642tQ7I`Uf$k0#CGtu8yzb!VkA{$NUETKD<}S^&}lXn1Gn7xL_=3U z5HU!UHza3CC1oX<*VYa1iZci3jaKXPVy{KJuExAf0$~y#8*_8dXlq!*o?AXqyEKM^ zxLdzTD+0HK1_y@(2L-O24C)`o^iY`m9xqWo++t?rGK(c$loUwHcJ z9ci>e92c3P(HFK%ax{j!gmJ$nX1_{KVkgxnCclJ)K(yx~!@5G8bjE+81IiF>{GC@N zDHgNEjYaCDAC;NRe|>G^cn3BzlN;^}>OROMWvsGBNm()2ZKN^=Q@xA6uywKnWG@?I z4L`ju5{pGbHU!mio_3bgFbfOhxqRnMunxsS*vDAlHphje8vspV4{$g?BluiQL74=i z_NG#mFCq5A7PvMF6W9!09hqKKhTQZf$Q@BzntIlM0#I)O`jC~+uIn)gD7zrZnDl5Q zT>54lx~L_lp!J%zcJnq!Rxl+UBHLDuvNu6)l?DiHqofCOaHZ6%Tq!tI%QT~Oo33i- zOE}}UCSP0`ZW0T9bt7WQoO7SH6!b=IQo1%xU7NVBO=Z_6w`m=YSu(rH>1;nG;^7VD|_;vI9+w%T(>jB!=3Fs9Ov@0=aSANjGHlbbF zLi>sa{fYjdg} ze7@48^9q&El(l@e7^Z{b87KhTsc!HE+Lx-KzNY#N_Ek-_++cM4KcQL2D z=vr!vM`YzC%ppg&ZllP{|3~hUt`oGb;ZfmOZh8iS84N{Jw1__F93^hn(DVCuI z$xwr&F0<5$F{x@F4K+xr`cs_;Q(&k;LiLu08YFG}QWKli6cZV0kPJ0QsFul4gJh^d zGSnbxDl`l=NZKkZvIeEC>d8=pq+5da$;0{U;w3{3lA#94P=jQsL4x`vLk*Il2C1X6 zDYAaaP=jQsLHbNl5WByj2FXx^WT-(h)F44+)8`6{3^hpV98XOd*ykSBf8bC<4U(Y- z$xwr2s6jH+Ad&xwn4t#AP=loTcV`SWNQN4u4hohGHAvcjfxu9MWT-*<|E@to6(8+0 Sul0Yo&$veaCryVL*Z&QhjuOTI literal 0 HcmV?d00001 diff --git a/docs/api.rst b/docs/api.rst index e16b9f18..4bc7c763 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -2,23 +2,15 @@ API Documentation ================= -.. automodule:: borg.key +.. automodule:: borg.archiver :members: :undoc-members: -.. automodule:: borg.cache +.. automodule:: borg.upgrader :members: :undoc-members: -.. automodule:: borg.locking - :members: - :undoc-members: - -.. automodule:: borg.platform - :members: - :undoc-members: - -.. automodule:: borg.xattr +.. automodule:: borg.archive :members: :undoc-members: @@ -26,7 +18,11 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.logger +.. automodule:: borg.platform + :members: + :undoc-members: + +.. automodule:: borg.locking :members: :undoc-members: @@ -34,14 +30,6 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.archiver - :members: - :undoc-members: - -.. automodule:: borg.archive - :members: - :undoc-members: - .. automodule:: borg.lrucache :members: :undoc-members: @@ -50,7 +38,7 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.upgrader +.. automodule:: borg.xattr :members: :undoc-members: @@ -58,15 +46,15 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.platform_freebsd +.. automodule:: borg.cache :members: :undoc-members: -.. automodule:: borg.hashindex +.. automodule:: borg.key :members: :undoc-members: -.. automodule:: borg.chunker +.. automodule:: borg.logger :members: :undoc-members: @@ -78,10 +66,22 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.hashindex + :members: + :undoc-members: + .. automodule:: borg.compress :members: :undoc-members: +.. automodule:: borg.chunker + :members: + :undoc-members: + .. automodule:: borg.crypto :members: :undoc-members: + +.. automodule:: borg.platform_freebsd + :members: + :undoc-members: diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index fb15b428..020881e4 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -6,7 +6,8 @@ borg check usage: borg check [-h] [-v] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] - [--repository-only] [--archives-only] [--repair] [--last N] + [--repository-only] [--archives-only] [--repair] + [--save-space] [--last N] [-p PREFIX] [REPOSITORY_OR_ARCHIVE] Check repository consistency @@ -30,7 +31,10 @@ borg check --repository-only only perform repository checks --archives-only only perform archives checks --repair attempt to repair any inconsistencies found + --save-space work slower, but using less space --last N only check last N archives (Default: all) + -p PREFIX, --prefix PREFIX + only consider archive names starting with this prefix Description ~~~~~~~~~~~ diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index f0068574..96460f2a 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -35,11 +35,10 @@ borg create --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -s, --stats print statistics for the created archive - -p, --progress, --no-progress - toggle progress display while creating the archive, + -p, --progress show progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path - being processed, default: True + being processed, default: False --filter STATUSCHARS only display items with the given status characters -e PATTERN, --exclude PATTERN exclude paths matching PATTERN diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index 587e7064..1c7642b9 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -6,7 +6,7 @@ borg delete usage: borg delete [-h] [-v] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [-s] - [-c] + [-c] [--save-space] [TARGET] Delete an existing repository or archive @@ -28,6 +28,7 @@ borg delete --remote-path PATH set remote path to executable (default: "borg") -s, --stats print statistics for the deleted archive -c, --cache-only delete only the local cache for the given repository + --save-space work slower, but using less space Description ~~~~~~~~~~~ diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index 08504b5d..d5253264 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -8,6 +8,7 @@ borg prune [--no-files-cache] [--umask M] [--remote-path PATH] [-n] [-s] [--keep-within WITHIN] [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY] [-p PREFIX] + [--save-space] [REPOSITORY] Prune repository archives according to specified rules @@ -42,6 +43,7 @@ borg prune number of yearly archives to keep -p PREFIX, --prefix PREFIX only consider archive names starting with this prefix + --save-space work slower, but using less space Description ~~~~~~~~~~~ From 393e36b6da91c45465d0ffb2eb8c040665768ba0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 19:58:32 +0100 Subject: [PATCH 167/321] updated internals docs: hash table max. load factor is 0.75 now --- docs/internals.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 2ebed0c5..5f3e96ec 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -276,10 +276,10 @@ buckets. As a consequence the hash is just a start position for a linear search, and if the element is not in the table the index is linearly crossed until an empty bucket is found. -When the hash table is almost full at 90%, its size is doubled. When it's -almost empty at 25%, its size is halved. So operations on it have a variable +When the hash table is filled to 75%, its size is doubled. When it's +emptied to 25%, its size is halved. So operations on it have a variable complexity between constant and linear with low factor, and memory overhead -varies between 10% and 300%. +varies between 33% and 300%. Indexes / Caches memory usage From c200b794707dd8f4200ef20eaa242b8fcda59ce3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 13 Dec 2015 21:34:37 +0100 Subject: [PATCH 168/321] development docs: run build_api and build_usage before tagging release --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 0d75dfe4..607abe6c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -164,11 +164,11 @@ Checklist: - update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..`` - check version number of upcoming release in ``CHANGES.rst`` - verify that ``MANIFEST.in`` and ``setup.py`` are complete +- ``python setup.py build_api ; python setup.py build_usage`` - tag the release:: git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z -- build fresh docs and update the web site with them - create a release on PyPi:: python setup.py register sdist upload --identity="Thomas Waldmann" --sign From f861f1f080cb1861df9c87e14a73680673c19368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 13 Dec 2015 15:59:33 -0500 Subject: [PATCH 169/321] rephrase the mailing list section --- docs/support.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/support.rst b/docs/support.rst index c2a8e389..02024b66 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -26,12 +26,10 @@ Stay connected. Mailing list ------------ -New Mailing List: - -`borgbackup@python.org `_ - -Just read that page to find out about the mailing list, its topic, how to subscribe, -how to unsubscribe and where you can find the archives of the list. +To find out about the mailing list, its topic, how to subscribe, how to +unsubscribe and where you can find the archives of the list, see the +`mailing list homepage +`_. Bounties and Fundraisers ------------------------ From 57b913bc883c76b1ad4525dd6c8bccce638a1c48 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 20 Dec 2015 02:03:33 +0100 Subject: [PATCH 170/321] fix badly named environment variable, fixes #503 added: BORG_DELETE_I_KNOW_WHAT_I_AM_DOING for the check in "borg delete" --- borg/archiver.py | 2 +- borg/testsuite/archiver.py | 1 + borg/testsuite/benchmark.py | 1 + docs/usage.rst | 2 ++ 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 57fa72a2..c07a324d 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -351,7 +351,7 @@ class Archiver: msg.append("Type 'YES' if you understand this and want to continue: ") msg = '\n'.join(msg) if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): self.exit_code = EXIT_ERROR return self.exit_code repository.destroy() diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index c20a5182..adb05e10 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -199,6 +199,7 @@ class ArchiverTestCaseBase(BaseTestCase): def setUp(self): os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1' + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1' self.archiver = not self.FORK_DEFAULT and Archiver() or None self.tmpdir = tempfile.mkdtemp() self.repository_path = os.path.join(self.tmpdir, 'repository') diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py index e7d9b248..3d59b672 100644 --- a/borg/testsuite/benchmark.py +++ b/borg/testsuite/benchmark.py @@ -17,6 +17,7 @@ from .archiver import changedir, cmd def repo_url(request, tmpdir): os.environ['BORG_PASSPHRASE'] = '123456' os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1' + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1' os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1' os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys')) os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache')) diff --git a/docs/usage.rst b/docs/usage.rst index ceea1cf5..9c3dcfff 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -72,6 +72,8 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do For "Warning: The repository at location ... was previously located at ..." BORG_CHECK_I_KNOW_WHAT_I_AM_DOING For "Warning: 'check --repair' is an experimental feature that might result in data loss." + BORG_DELETE_I_KNOW_WHAT_I_AM_DOING + For "You requested to completely DELETE the repository *including* all archives it contains: " Directories: BORG_KEYS_DIR From 546052ed921ae2caf20e3443af9bbb5a42a4efa0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 20 Dec 2015 16:38:38 +0100 Subject: [PATCH 171/321] README: minor grammar fix, minor other changes --- README.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 47e2f1c0..2a681479 100644 --- a/README.rst +++ b/README.rst @@ -73,9 +73,8 @@ Main features backup examination and restores (e.g. by using a regular file manager). **Easy installation on multiple platforms** - We offer single-file binaries - that does not require installing anything - you can just run it on - the supported platforms: + We offer single-file binaries that do not require installing anything - + you can just run them on these platforms: * Linux * Mac OS X @@ -148,7 +147,7 @@ Here's a (incomplete) list of some major changes: * mkdir-based locking is more compatible than attic's posix locking * uses fadvise to not spoil / blow up the fs cache * better error messages / exception handling - * better output for verbose mode, progress indication + * better logging, screen output, progress indication * tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X Please read the `ChangeLog`_ (or ``CHANGES.rst`` in the source distribution) for more From c9afa2b27b930558e6ce0da1787079e4ffe93121 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Dec 2015 11:06:03 +0100 Subject: [PATCH 172/321] output progress indication from inner loop, fixes #500 - so it shows progress while it backups a bigger file - so it announces the filename earlier also: move rate limiting code to show_progress() --- borg/archive.py | 11 ++++++----- borg/helpers.py | 30 +++++++++++++++++------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 390c64ff..3edd4522 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -143,7 +143,6 @@ class Archive: self.hard_links = {} self.stats = Statistics() self.show_progress = progress - self.last_progress = time.time() self.name = name self.checkpoint_interval = checkpoint_interval self.numeric_owner = numeric_owner @@ -215,9 +214,8 @@ Number of files: {0.stats.nfiles}'''.format(self) unknown_keys = set(item) - ITEM_KEYS assert not unknown_keys, ('unknown item metadata keys detected, please update ITEM_KEYS: %s', ','.join(k.decode('ascii') for k in unknown_keys)) - if self.show_progress and time.time() - self.last_progress > 0.2: - self.stats.show_progress(item=item) - self.last_progress = time.time() + if self.show_progress: + self.stats.show_progress(item=item, dt=0.2) self.items_buffer.add(item) if time.time() - self.last_checkpoint > self.checkpoint_interval: self.write_checkpoint() @@ -526,6 +524,7 @@ Number of files: {0.stats.nfiles}'''.format(self) status = 'U' # regular file, unchanged else: status = 'A' # regular file, added + item = {b'path': safe_path} # Only chunkify the file if needed if chunks is None: fh = Archive._open_rb(path, st) @@ -533,9 +532,11 @@ Number of files: {0.stats.nfiles}'''.format(self) chunks = [] for chunk in self.chunker.chunkify(fd, fh): chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)) + if self.show_progress: + self.stats.show_progress(item=item, dt=0.2) cache.memorize_file(path_hash, st, [c[0] for c in chunks]) status = status or 'M' # regular file, modified (if not 'A' already) - item = {b'path': safe_path, b'chunks': chunks} + item[b'chunks'] = chunks item.update(self.stat_attrs(st, path)) self.stats.nfiles += 1 self.add_item(item) diff --git a/borg/helpers.py b/borg/helpers.py index 925dfb11..8e76c362 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -162,6 +162,7 @@ class Statistics: def __init__(self): self.osize = self.csize = self.usize = self.nfiles = 0 + self.last_progress = 0 # timestamp when last progress was shown def update(self, size, csize, unique): self.osize += size @@ -191,19 +192,22 @@ class Statistics: def csize_fmt(self): return format_file_size(self.csize) - def show_progress(self, item=None, final=False, stream=None): - columns, lines = get_terminal_size() - if not final: - msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self) - path = remove_surrogates(item[b'path']) if item else '' - space = columns - len(msg) - if space < len('...') + len(path): - path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:]) - msg += "{0:<{space}}".format(path, space=space) - else: - msg = ' ' * columns - print(msg, file=stream or sys.stderr, end="\r") - (stream or sys.stderr).flush() + def show_progress(self, item=None, final=False, stream=None, dt=None): + now = time.time() + if dt is None or now - self.last_progress > dt: + self.last_progress = now + columns, lines = get_terminal_size() + if not final: + msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self) + path = remove_surrogates(item[b'path']) if item else '' + space = columns - len(msg) + if space < len('...') + len(path): + path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:]) + msg += "{0:<{space}}".format(path, space=space) + else: + msg = ' ' * columns + print(msg, file=stream or sys.stderr, end="\r") + (stream or sys.stderr).flush() def get_keys_dir(): From 4ed71e2cf5ef373878da68b921c636d6403d9f2d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 27 Dec 2015 14:10:41 +0100 Subject: [PATCH 173/321] add some error handling/fallback for C library loading, fixes #494 --- borg/xattr.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/borg/xattr.py b/borg/xattr.py index 9c80c326..7eafaa83 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -7,6 +7,9 @@ import tempfile from ctypes import CDLL, create_string_buffer, c_ssize_t, c_size_t, c_char_p, c_int, c_uint32, get_errno from ctypes.util import find_library +from .logger import create_logger +logger = create_logger() + def is_enabled(path=None): """Determine if xattr is enabled on the filesystem @@ -27,8 +30,28 @@ def get_all(path, follow_symlinks=True): if e.errno in (errno.ENOTSUP, errno.EPERM): return {} +libc_name = find_library('c') +if libc_name is None: + # find_library didn't work, maybe we are on some minimal system that misses essential + # tools used by find_library, like ldconfig, gcc/cc, objdump. + # so we can only try some "usual" names for the C library: + if sys.platform.startswith('linux'): + libc_name = 'libc.so.6' + elif sys.platform.startswith(('freebsd', 'netbsd')): + libc_name = 'libc.so' + elif sys.platform == 'darwin': + libc_name = 'libc.dylib' + else: + msg = "Can't find C library. No fallback known. Try installing ldconfig, gcc/cc or objdump." + logger.error(msg) + raise Exception(msg) -libc = CDLL(find_library('c'), use_errno=True) +try: + libc = CDLL(libc_name, use_errno=True) +except OSError as e: + msg = "Can't find C library [%s]. Try installing ldconfig, gcc/cc or objdump." % e + logger.error(msg) + raise Exception(msg) def _check(rv, path=None): From eb642f06cc8ed67917a3e42c4ad38d4ce727cd12 Mon Sep 17 00:00:00 2001 From: Fabian Weisshaar Date: Wed, 30 Dec 2015 15:06:31 +0100 Subject: [PATCH 174/321] Allow simple copy-paste for package installation with apt --- docs/installation.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 6441db79..13de6ae0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -109,11 +109,11 @@ Debian / Ubuntu Install the dependencies with development headers:: - sudo apt-get install python3 python3-dev python3-pip python-virtualenv - sudo apt-get install libssl-dev openssl - sudo apt-get install libacl1-dev libacl1 - sudo apt-get install liblz4-dev liblz4-1 - sudo apt-get install build-essential + sudo apt-get install python3 python3-dev python3-pip python-virtualenv \ + libssl-dev openssl \ + libacl1-dev libacl1 \ + liblz4-dev liblz4-1 \ + build-essential sudo apt-get install libfuse-dev fuse pkg-config # optional, for FUSE support In case you get complaints about permission denied on ``/etc/fuse.conf``: on From fb64173bb411c12f63a339c1574dca31666c477b Mon Sep 17 00:00:00 2001 From: Michael Gajda Date: Wed, 6 Jan 2016 13:35:28 +0100 Subject: [PATCH 175/321] Documentation: Standalone binary / pyinstaller extracts dependencies into /tmp. Currently /tmp requires about ~28MB of free space. It also needs exec permissions. Closes #499 --- docs/installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 13de6ae0..7a007e31 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -69,6 +69,8 @@ make borg readable and executable for its users and then you can run ``borg``:: sudo chown root:root /usr/local/bin/borg sudo chmod 755 /usr/local/bin/borg +Note that the binary uses /tmp to unpack |project_name| with all dependencies. It will fail if /tmp has not enough free space or is mounted with the ``noexec`` option. You can change the temporary directory by setting the ``TEMP`` environment variable before running |project_name|. + If a new version is released, you will have to manually download it and replace the old version using the same steps as shown above. From d668901df4491d148daf6a7aa0cea3eac194a248 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Thu, 7 Jan 2016 11:05:28 +0100 Subject: [PATCH 176/321] Fix typo in comment --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index 8e76c362..7f20a102 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -229,7 +229,7 @@ def to_localtime(ts): def parse_timestamp(timestamp): """Parse a ISO 8601 timestamp string""" - if '.' in timestamp: # microseconds might not be pressent + if '.' in timestamp: # microseconds might not be present return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc) else: return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) From 077ebe8c4981549f2f64a7a809961e6a487a8530 Mon Sep 17 00:00:00 2001 From: Leo Famulari Date: Fri, 8 Jan 2016 21:02:03 -0500 Subject: [PATCH 177/321] docs: Give project name in usage example. * docs/usage.rst: Replace "|project_name|" with "borg" because the abstraction doesn't work in usage examples. --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 9c3dcfff..a3f4bfa2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -409,7 +409,7 @@ Examples ~~~~~~~~ :: - # Allow an SSH keypair to only run |project_name|, and only have access to /mnt/backup. + # Allow an SSH keypair to only run borg, and only have access to /mnt/backup. # This will help to secure an automated remote backup system. $ cat ~/.ssh/authorized_keys command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...] From 0c2e517e04fa5e8fb5f91583d3eed6a81c3bb85c Mon Sep 17 00:00:00 2001 From: Hartmut Goebel Date: Sat, 9 Jan 2016 23:50:41 +0100 Subject: [PATCH 178/321] Update FAQ Clarify that user and group of owner are stored as name, except if --numeric-owner is given. --- docs/faq.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index ef6ca661..bebb291f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -38,8 +38,8 @@ Which file types, attributes, etc. are preserved? * Name * Contents * Time of last modification (nanosecond precision with Python >= 3.3) - * User ID of owner - * Group ID of owner + * IDs of owning user and owning group + * Names of owning user and owning group (if the IDs can be resolved) * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) * Extended Attributes (xattrs) on Linux, OS X and FreeBSD * Access Control Lists (ACL_) on Linux, OS X and FreeBSD From 170f847e74e7909ef601e8245d0ed1bada9a2e2d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jan 2016 02:08:58 +0100 Subject: [PATCH 179/321] unset LD_LIBRARY_PATH before invoking ssh, hopefully fixes #514 --- borg/remote.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/borg/remote.py b/borg/remote.py index 1fcd97c5..0c572cd9 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -141,9 +141,13 @@ class RemoteRepository: self.p = None testing = location.host == '__testsuite__' borg_cmd = self.borg_cmd(args, testing) + env = dict(os.environ) if not testing: borg_cmd = self.ssh_cmd(location) + borg_cmd - self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE) + # pyinstaller binary adds LD_LIBRARY_PATH=/tmp/_ME... but we do not want + # that the system's ssh binary picks up (non-matching) libraries from there + env.pop('LD_LIBRARY_PATH', None) + self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env) self.stdin_fd = self.p.stdin.fileno() self.stdout_fd = self.p.stdout.fileno() self.stderr_fd = self.p.stderr.fileno() From 02e04653b613b55d66029deef0b0f8c95190064c Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Thu, 7 Jan 2016 11:24:47 +0100 Subject: [PATCH 180/321] Factorize and test loading of excludes from file The parsing code for exclude files (given via `--exclude-from`) was not tested. Its core is factorized into a separate function to facilitate an easier test. The observable behaviour is unchanged. --- borg/helpers.py | 15 ++++++++++----- borg/testsuite/helpers.py | 37 ++++++++++++++++++++++++++++++++++++- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 7f20a102..62b32781 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -235,16 +235,21 @@ def parse_timestamp(timestamp): return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) +def load_excludes(fh): + """Load and parse exclude patterns from file object. Empty lines and lines starting with '#' are ignored, but + whitespace is not stripped. + """ + patterns = (line.rstrip('\r\n') for line in fh if not line.startswith('#')) + return [ExcludePattern(pattern) for pattern in patterns if pattern] + + def update_excludes(args): - """Merge exclude patterns from files with those on command line. - Empty lines and lines starting with '#' are ignored, but whitespace - is not stripped.""" + """Merge exclude patterns from files with those on command line.""" if hasattr(args, 'exclude_files') and args.exclude_files: if not hasattr(args, 'excludes') or args.excludes is None: args.excludes = [] for file in args.exclude_files: - patterns = [line.rstrip('\r\n') for line in file if not line.startswith('#')] - args.excludes += [ExcludePattern(pattern) for pattern in patterns if pattern] + args.excludes += load_excludes(file) file.close() diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 9556b5e8..e97c4ed3 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -12,7 +12,7 @@ import msgpack.fallback from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ - ProgressIndicatorPercent, ProgressIndicatorEndless + ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes from . import BaseTestCase, environment_variable, FakeInputs @@ -259,6 +259,41 @@ class OSXPatternNormalizationTestCase(BaseTestCase): assert e.match(str(b"ba\x80/foo", 'latin1')) +@pytest.mark.parametrize("lines, expected", [ + # "None" means all files, i.e. none excluded + ([], None), + (["# Comment only"], None), + (["*"], []), + (["# Comment", + "*/something00.txt", + " whitespace\t", + "/whitespace/at/end of filename \t ", + # Whitespace before comment + " #/ws*", + # Empty line + "", + "# EOF"], + ["/more/data", "/home"]), + ]) +def test_patterns_from_file(tmpdir, lines, expected): + files = [ + '/data/something00.txt', '/more/data', '/home', + ' #/wsfoobar', + '/whitespace/at/end of filename \t ', + ] + + def evaluate(filename): + patterns = load_excludes(open(filename, "rt")) + return [path for path in files if not exclude_path(path, patterns)] + + exclfile = tmpdir.join("exclude.txt") + + with exclfile.open("wt") as fh: + fh.write("\n".join(lines)) + + assert evaluate(str(exclfile)) == (files if expected is None else expected) + + def test_compression_specs(): with pytest.raises(ValueError): CompressionSpec('') From 857f563307b17afababc63063b2bb24983ce0a46 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jan 2016 23:22:04 +0100 Subject: [PATCH 181/321] display borg version below tracebacks, fixes #532 --- borg/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 62b32781..9fd709ad 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -22,7 +22,7 @@ from datetime import datetime, timezone, timedelta from fnmatch import translate from operator import attrgetter - +from . import __version__ as borg_version from . import hashindex from . import chunker from . import crypto @@ -968,6 +968,6 @@ def sysinfo(): info.append('Platform: %s' % (' '.join(platform.uname()), )) if sys.platform.startswith('linux'): info.append('Linux: %s %s %s LibC: %s %s' % (platform.linux_distribution() + platform.libc_ver())) - info.append('Python: %s %s' % (platform.python_implementation(), platform.python_version())) + info.append('Borg: %s Python: %s %s' % (borg_version, platform.python_implementation(), platform.python_version())) info.append('') return '\n'.join(info) From e5c29bd145ed499b96ec4515dca9ceed385133a4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jan 2016 23:31:24 +0100 Subject: [PATCH 182/321] add abbreviated weekday to timestamp format, fixes #496 --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index 62b32781..57f0a702 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -462,7 +462,7 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): def format_time(t): """use ISO-8601 date and time format """ - return t.strftime('%Y-%m-%d %H:%M:%S') + return t.strftime('%a, %Y-%m-%d %H:%M:%S') def format_timedelta(td): From 84672f7081408111fb00fffb96872cdd2cb7a492 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Jan 2016 00:41:06 +0100 Subject: [PATCH 183/321] log stats consistently, fixes #526 prune and create now both require --verbose --stats to show stats. it was implemented in this way (and not with print) so you can feed the stats data into the logging system, too. delete now says "Archive deleted" in verbose mode (for consistency, it already said "Repository deleted" when deleting a repo). also: add helpers.log_multi to comfortably and prettily output a block of log lines --- borg/archiver.py | 29 ++++++++++++++++++----------- borg/helpers.py | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index c07a324d..9a175c3b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -21,7 +21,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ - EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR + EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi from .logger import create_logger, setup_logging logger = create_logger() from .compress import Compressor, COMPR_BUFFER @@ -37,6 +37,8 @@ has_lchflags = hasattr(os, 'lchflags') # default umask, overriden by --umask, defaults to read/write only for owner UMASK_DEFAULT = 0o077 +DASHES = '-' * 78 + class ToggleAction(argparse.Action): """argparse action to handle "toggle" flags easily @@ -187,12 +189,12 @@ class Archiver: archive.stats.show_progress(final=True) if args.stats: archive.end = datetime.now() - print('-' * 78) - print(str(archive)) - print() - print(str(archive.stats)) - print(str(cache)) - print('-' * 78) + log_multi(DASHES, + str(archive), + DASHES, + str(archive.stats), + str(cache), + DASHES) return self.exit_code def _process(self, archive, cache, excludes, exclude_caches, exclude_if_present, @@ -339,9 +341,12 @@ class Archiver: manifest.write() repository.commit(save_space=args.save_space) cache.commit() + logger.info("Archive deleted.") if args.stats: - logger.info(stats.summary.format(label='Deleted data:', stats=stats)) - logger.info(str(cache)) + log_multi(DASHES, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES) else: if not args.cache_only: msg = [] @@ -495,8 +500,10 @@ class Archiver: repository.commit(save_space=args.save_space) cache.commit() if args.stats: - logger.info(stats.summary.format(label='Deleted data:', stats=stats)) - logger.info(str(cache)) + log_multi(DASHES, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES) return self.exit_code def do_upgrade(self, args): diff --git a/borg/helpers.py b/borg/helpers.py index 62b32781..5afadbb5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -18,6 +18,10 @@ import platform import time import unicodedata +import logging +from .logger import create_logger +logger = create_logger() + from datetime import datetime, timezone, timedelta from fnmatch import translate from operator import attrgetter @@ -971,3 +975,16 @@ def sysinfo(): info.append('Python: %s %s' % (platform.python_implementation(), platform.python_version())) info.append('') return '\n'.join(info) + + +def log_multi(*msgs, level=logging.INFO): + """ + log multiple lines of text, each line by a separate logging call for cosmetic reasons + + each positional argument may be a single or multiple lines (separated by \n) of text. + """ + lines = [] + for msg in msgs: + lines.extend(msg.splitlines()) + for line in lines: + logger.log(level, line) From 98da9d1b9609a53ef769a3f9cedb691037eb83d8 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Tue, 12 Jan 2016 12:31:01 +0100 Subject: [PATCH 184/321] Dedent pattern help text The help text describing patterns should be dedented like other multi-paragraph text blocks. --- borg/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index c07a324d..e3abe102 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -597,7 +597,7 @@ class Archiver: return self.exit_code helptext = {} - helptext['patterns'] = ''' + helptext['patterns'] = textwrap.dedent(''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any number of characters, '?' matching any single character, '[...]' matching any single character specified, including ranges, and '[!...]' matching any @@ -624,7 +624,7 @@ class Archiver: # The file '/home/user/cache/important' is *not* backed up: $ borg create -e /home/user/cache/ backup / /home/user/cache/important - ''' + ''') def do_help(self, parser, commands, args): if not args.topic: From 0f4d3b21c3f085e883f832d5181f2e9c9f7e5c51 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Jan 2016 23:49:19 +0100 Subject: [PATCH 185/321] minor development docs fixes --- docs/development.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 607abe6c..42036950 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -4,7 +4,7 @@ Development =========== -This chapter will get you started with |project_name|' development. +This chapter will get you started with |project_name| development. |project_name| is written in Python (with a little bit of Cython and C for the performance critical parts). @@ -164,7 +164,7 @@ Checklist: - update ``CHANGES.rst``, based on ``git log $PREVIOUS_RELEASE..`` - check version number of upcoming release in ``CHANGES.rst`` - verify that ``MANIFEST.in`` and ``setup.py`` are complete -- ``python setup.py build_api ; python setup.py build_usage`` +- ``python setup.py build_api ; python setup.py build_usage`` and commit - tag the release:: git tag -s -m "tagged/signed release X.Y.Z" X.Y.Z @@ -181,7 +181,9 @@ Checklist: - IRC channel (change ``/topic``) - create a Github release, include: + * standalone binaries (see above for how to create them) + + for OS X, document the OS X Fuse version in the README of the binaries. OS X FUSE uses a kernel extension that needs to be compatible with the code contained in the binary. From 9a2d1eb1d80e2ba043422c1cb4b18b3eaa4db413 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Jan 2016 00:25:43 +0100 Subject: [PATCH 186/321] docs: replace "|project_name|" with just "Borg", less ugly --- docs/global.rst.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/global.rst.inc b/docs/global.rst.inc index eaa4648b..317c8d85 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -1,5 +1,5 @@ .. highlight:: bash -.. |project_name| replace:: ``Borg`` +.. |project_name| replace:: Borg .. |package_dirname| replace:: borgbackup-|version| .. |package_filename| replace:: |package_dirname|.tar.gz .. |package_url| replace:: https://pypi.python.org/packages/source/b/borgbackup/|package_filename| From 86ec3847e20a6d7021c360db42336ecd0dbeebe7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Jan 2016 00:36:14 +0100 Subject: [PATCH 187/321] Authors: make it more clear what refers to borg and what to attic With some sphinx output formats (e.g. "man") that was not too clear before this change. --- AUTHORS | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/AUTHORS b/AUTHORS index 6812638d..31910fb6 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,23 +1,24 @@ -Contributors ("The Borg Collective") -==================================== +Borg Contributors ("The Borg Collective") +========================================= - Thomas Waldmann - Antoine Beaupré - Radek Podgorny - Yuri D'Elia +Borg is a fork of Attic. + Attic authors ------------- -Borg is a fork of Attic. Attic is written and maintained -by Jonas Borgström and various contributors: +Attic is written and maintained by Jonas Borgström and various contributors: -Development Lead -```````````````` +Attic Development Lead +`````````````````````` - Jonas Borgström -Patches and Suggestions -``````````````````````` +Attic Patches and Suggestions +````````````````````````````` - Brian Johnson - Cyril Roussillon - Dan Christensen From 4216a94e19cc2e2b6696136d5af269fd09006ce8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Jan 2016 00:42:23 +0100 Subject: [PATCH 188/321] it's 2016 --- LICENSE | 2 +- docs/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index ad958c54..251e7027 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2015 The Borg Collective (see AUTHORS file) +Copyright (C) 2015-2016 The Borg Collective (see AUTHORS file) Copyright (C) 2010-2014 Jonas Borgström All rights reserved. diff --git a/docs/conf.py b/docs/conf.py index 9c862cc5..882543a1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -44,7 +44,7 @@ master_doc = 'index' # General information about the project. project = 'Borg - Deduplicating Archiver' -copyright = '2010-2014 Jonas Borgström, 2015 The Borg Collective (see AUTHORS file)' +copyright = '2010-2014 Jonas Borgström, 2015-2016 The Borg Collective (see AUTHORS file)' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 7420ea00339199294f5f76465d1b217417e54505 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 13 Jan 2016 01:18:07 +0100 Subject: [PATCH 189/321] sphinx configuration: fix to create a simple man page from usage docs --- docs/conf.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 882543a1..72eb833a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -228,10 +228,12 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -#man_pages = [ -# ('man', 'borg', 'Borg', -# ['see "AUTHORS" file'], 1) -#] +man_pages = [ + ('usage', 'borg', + 'BorgBackup is a deduplicating backup program with optional compression and authenticated encryption.', + ['The Borg Collective (see AUTHORS file)'], + 1), +] extensions = ['sphinx.ext.extlinks', 'sphinx.ext.autodoc', 'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.viewcode'] From 5d40eba175e8b3d9f347b9ed3b2f15cedda67308 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Tue, 12 Jan 2016 10:04:01 +0100 Subject: [PATCH 190/321] Convert pattern test to py.test The test for exclusion patterns was written using the standard unittest module. The py.test module provides facilities to parametrize the test. --- borg/testsuite/helpers.py | 43 ++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index e97c4ed3..e766ed44 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -160,8 +160,23 @@ class FormatTimedeltaTestCase(BaseTestCase): ) -class PatternTestCase(BaseTestCase): - +@pytest.mark.parametrize("paths, excludes, expected", [ + # "None" means all files, i.e. none excluded + ([], [], None), + (['/'], [], None), + (['/'], ['/h'], None), + (['/'], ['/home'], ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']), + (['/'], ['/home/'], ['/etc/passwd', '/etc/hosts', '/home', '/var/log/messages', '/var/log/dmesg']), + (['/home/u'], [], []), + (['/', '/home', '/etc/hosts'], ['/'], []), + (['/home/'], ['/home/user2'], ['/home', '/home/user/.profile', '/home/user/.bashrc']), + (['/'], ['*.profile', '/var/log'], + ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc', '/home/user2/public_html/index.html']), + (['/'], ['/home/*/public_html', '*.profile', '*/log/*'], + ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc']), + (['/etc/', '/var'], ['dmesg'], ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']), + ]) +def test_patterns(paths, excludes, expected): files = [ '/etc/passwd', '/etc/hosts', '/home', '/home/user/.profile', '/home/user/.bashrc', @@ -169,28 +184,10 @@ class PatternTestCase(BaseTestCase): '/var/log/messages', '/var/log/dmesg', ] - def evaluate(self, paths, excludes): - patterns = adjust_patterns(paths, [ExcludePattern(p) for p in excludes]) - return [path for path in self.files if not exclude_path(path, patterns)] + patterns = adjust_patterns(paths, [ExcludePattern(p) for p in excludes]) + included = [path for path in files if not exclude_path(path, patterns)] - def test(self): - self.assert_equal(self.evaluate(['/'], []), self.files) - self.assert_equal(self.evaluate([], []), self.files) - self.assert_equal(self.evaluate(['/'], ['/h']), self.files) - self.assert_equal(self.evaluate(['/'], ['/home']), - ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']) - self.assert_equal(self.evaluate(['/'], ['/home/']), - ['/etc/passwd', '/etc/hosts', '/home', '/var/log/messages', '/var/log/dmesg']) - self.assert_equal(self.evaluate(['/home/u'], []), []) - self.assert_equal(self.evaluate(['/', '/home', '/etc/hosts'], ['/']), []) - self.assert_equal(self.evaluate(['/home/'], ['/home/user2']), - ['/home', '/home/user/.profile', '/home/user/.bashrc']) - self.assert_equal(self.evaluate(['/'], ['*.profile', '/var/log']), - ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc', '/home/user2/public_html/index.html']) - self.assert_equal(self.evaluate(['/'], ['/home/*/public_html', '*.profile', '*/log/*']), - ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc']) - self.assert_equal(self.evaluate(['/etc/', '/var'], ['dmesg']), - ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']) + assert included == (files if expected is None else expected) @pytest.mark.skipif(sys.platform in ('darwin',), reason='all but OS X test') From 93c9c492501a95d343f5257a580b50ae843cc3a6 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Wed, 13 Jan 2016 14:26:50 +0100 Subject: [PATCH 191/321] Reduce code duplication in inclusion/exclusion pattern logic The two classes for applying inclusion and exclusion patterns contained unnecessarily duplicated logic. The introduction of a shared base class allows for easier reuse, especially considering that two more classes are going to be added in forthcoming changes (regular expressions and shell-style patterns). --- borg/helpers.py | 73 ++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index f426bb95..d994ab25 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -270,12 +270,6 @@ def exclude_path(path, patterns): return False -# For both IncludePattern and ExcludePattern, we require that -# the pattern either match the whole path or an initial segment -# of the path up to but not including a path separator. To -# unify the two cases, we add a path separator to the end of -# the path before matching. - def normalized(func): """ Decorator for the Pattern match methods, returning a wrapper that normalizes OSX paths to match the normalized pattern on OSX, and @@ -294,11 +288,8 @@ def normalized(func): return func -class IncludePattern: - """Literal files or directories listed on the command line - for some operations (e.g. extract, but not create). - If a directory is specified, all paths that start with that - path match as well. A trailing slash makes no difference. +class PatternBase: + """Shared logic for inclusion/exclusion patterns. """ def __init__(self, pattern): self.pattern_orig = pattern @@ -307,13 +298,15 @@ class IncludePattern: if sys.platform in ('darwin',): pattern = unicodedata.normalize("NFD", pattern) - self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep + self._prepare(pattern) @normalized def match(self, path): - matches = (path+os.path.sep).startswith(self.pattern) + matches = self._match(path) + if matches: self.match_count += 1 + return matches def __repr__(self): @@ -322,39 +315,51 @@ class IncludePattern: def __str__(self): return self.pattern_orig + def _prepare(self, pattern): + raise NotImplementedError -class ExcludePattern(IncludePattern): + def _match(self, path): + raise NotImplementedError + + +# For both IncludePattern and ExcludePattern, we require that +# the pattern either match the whole path or an initial segment +# of the path up to but not including a path separator. To +# unify the two cases, we add a path separator to the end of +# the path before matching. + + +class IncludePattern(PatternBase): + """Literal files or directories listed on the command line + for some operations (e.g. extract, but not create). + If a directory is specified, all paths that start with that + path match as well. A trailing slash makes no difference. + """ + def _prepare(self, pattern): + self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + + def _match(self, path): + return (path + os.path.sep).startswith(self.pattern) + + +class ExcludePattern(PatternBase): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ - def __init__(self, pattern): - self.pattern_orig = pattern - self.match_count = 0 - + def _prepare(self, pattern): if pattern.endswith(os.path.sep): - self.pattern = os.path.normpath(pattern).rstrip(os.path.sep)+os.path.sep+'*'+os.path.sep + pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep else: - self.pattern = os.path.normpath(pattern)+os.path.sep+'*' + pattern = os.path.normpath(pattern) + os.path.sep+'*' - if sys.platform in ('darwin',): - self.pattern = unicodedata.normalize("NFD", self.pattern) + self.pattern = pattern # fnmatch and re.match both cache compiled regular expressions. # Nevertheless, this is about 10 times faster. self.regex = re.compile(translate(self.pattern)) - @normalized - def match(self, path): - matches = self.regex.match(path+os.path.sep) is not None - if matches: - self.match_count += 1 - return matches - - def __repr__(self): - return '%s(%s)' % (type(self), self.pattern) - - def __str__(self): - return self.pattern_orig + def _match(self, path): + return (self.regex.match(path + os.path.sep) is not None) def timestamp(s): From 2bafece093b6c261f5124f59e9f1c2e53c68bd61 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Wed, 16 Dec 2015 00:14:02 +0100 Subject: [PATCH 192/321] Implement exclusions using regular expressions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing option to exclude files and directories, “--exclude”, is implemented using fnmatch[1]. fnmatch matches the slash (“/”) with “*” and thus makes it impossible to write patterns where a directory with a given name should be excluded at a specific depth in the directory hierarchy, but not anywhere else. Consider this structure: home/ home/aaa home/aaa/.thumbnails home/user home/user/img home/user/img/.thumbnails fnmatch incorrectly excludes “home/user/img/.thumbnails” with a pattern of “home/*/.thumbnails” when the intention is to exclude “.thumbnails” in all home directories while retaining directories with the same name in all other locations. With this change regular expressions are introduced as an additional pattern syntax. The syntax is selected using a prefix on “--exclude”'s value. “re:” is for regular expression and “fm:”, the default, selects fnmatch. Selecting the syntax is necessary when regular expressions are desired or when the desired fnmatch pattern starts with two alphanumeric characters followed by a colon (i.e. “aa:something/*”). The exclusion described above can be implemented as follows: --exclude 're:^home/[^/]+/\.thumbnails$' The “--exclude-from” option permits loading exclusions from a text file where the same prefixes can now be used, e.g. “re:\.tmp$”. The documentation has been extended and now not only describes the two pattern styles, but also the file format supported by “--exclude-from”. This change has been discussed in issue #43 and in change request #497. [1] https://docs.python.org/3/library/fnmatch.html Signed-off-by: Michael Hanselmann --- borg/archiver.py | 68 ++++++++++++++++++----- borg/helpers.py | 42 +++++++++++++- borg/testsuite/archiver.py | 73 ++++++++++++++++++++++++ borg/testsuite/helpers.py | 111 +++++++++++++++++++++++++++++++++++-- docs/usage.rst | 5 ++ 5 files changed, 278 insertions(+), 21 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index e3abe102..beee1605 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,7 +17,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ + format_file_mode, parse_pattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ @@ -598,17 +598,43 @@ class Archiver: helptext = {} helptext['patterns'] = textwrap.dedent(''' - Exclude patterns use a variant of shell pattern syntax, with '*' matching any - number of characters, '?' matching any single character, '[...]' matching any - single character specified, including ranges, and '[!...]' matching any - character not specified. For the purpose of these patterns, the path - separator ('\\' for Windows and '/' on other systems) is not treated - specially. For a path to match a pattern, it must completely match from - start to end, or must match from the start to just before a path separator. - Except for the root path, paths will never end in the path separator when - matching is attempted. Thus, if a given pattern ends in a path separator, a - '*' is appended before matching is attempted. Patterns with wildcards should - be quoted to protect them from shell expansion. + Exclusion patterns support two separate styles, fnmatch and regular + expressions. If followed by a colon (':') the first two characters of + a pattern are used as a style selector. Explicit style selection is necessary + when regular expressions are desired or when the desired fnmatch pattern + starts with two alphanumeric characters followed by a colon (i.e. + `aa:something/*`). + + `Fnmatch `_ patterns use + a variant of shell pattern syntax, with '*' matching any number of + characters, '?' matching any single character, '[...]' matching any single + character specified, including ranges, and '[!...]' matching any character + not specified. The style selector is `fm`. For the purpose of these patterns, + the path separator ('\\' for Windows and '/' on other systems) is not treated + specially. For a path to match a pattern, it must completely match from start + to end, or must match from the start to just before a path separator. Except + for the root path, paths will never end in the path separator when matching + is attempted. Thus, if a given pattern ends in a path separator, a '*' is + appended before matching is attempted. + + Regular expressions similar to those found in Perl are supported with the + selection prefix `re:`. Unlike shell patterns regular expressions are not + required to match the complete path and any substring match is sufficient. It + is strongly recommended to anchor patterns to the start ('^'), to the end + ('$') or both. Path separators ('\\' for Windows and '/' on other systems) in + paths are always normalized to a forward slash ('/') before applying + a pattern. The regular expression syntax is described in the `Python + documentation for the re module + `_. + + Exclusions can be passed via the command line option `--exclude`. When used + from within a shell the patterns should be quoted to protect them from + expansion. + + The `--exclude-from` option permits loading exclusion patterns from a text + file with one pattern per line. Empty lines as well as lines starting with + the number sign ('#') are ignored. The optional style selector prefix is + also supported for patterns loaded from a file. Examples: @@ -624,6 +650,20 @@ class Archiver: # The file '/home/user/cache/important' is *not* backed up: $ borg create -e /home/user/cache/ backup / /home/user/cache/important + + # The contents of directories in '/home' are not backed up when their name + # ends in '.tmp' + $ borg create --exclude 're:^/home/[^/]+\.tmp/' backup / + + # Load exclusions from file + $ cat >exclude.txt < 2 and pattern[2] == ":" and pattern[:2].isalnum(): + (style, pattern) = (pattern[:2], pattern[3:]) + else: + style = _DEFAULT_PATTERN_STYLE + + cls = _PATTERN_STYLES.get(style, None) + + if cls is None: + raise ValueError("Unknown pattern style: {}".format(style)) + + return cls(pattern) + + def timestamp(s): """Convert a --timestamp=s argument to a datetime object""" try: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index adb05e10..58d20c52 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -489,6 +489,79 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file3']) + def test_extract_include_exclude_regex(self): + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('file2', size=1024 * 80) + self.create_regular_file('file3', size=1024 * 80) + self.create_regular_file('file4', size=1024 * 80) + self.create_regular_file('file333', size=1024 * 80) + + # Create with regular expression exclusion for file4 + self.cmd('create', '--exclude=re:input/file4$', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) + shutil.rmtree('output/input') + + # Extract with regular expression exclusion + with changedir('output'): + self.cmd('extract', '--exclude=re:file3+', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) + shutil.rmtree('output/input') + + # Combine --exclude with fnmatch and regular expression + with changedir('output'): + self.cmd('extract', '--exclude=input/file2', '--exclude=re:file[01]', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file3', 'file333']) + shutil.rmtree('output/input') + + # Combine --exclude-from and regular expression exclusion + with changedir('output'): + self.cmd('extract', '--exclude-from=' + self.exclude_file_path, '--exclude=re:file1', + '--exclude=re:file(\\d)\\1\\1$', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file3']) + + def test_extract_include_exclude_regex_from_file(self): + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('file2', size=1024 * 80) + self.create_regular_file('file3', size=1024 * 80) + self.create_regular_file('file4', size=1024 * 80) + self.create_regular_file('file333', size=1024 * 80) + self.create_regular_file('aa:something', size=1024 * 80) + + # Create while excluding using mixed pattern styles + with open(self.exclude_file_path, 'wb') as fd: + fd.write(b're:input/file4$\n') + fd.write(b'fm:*aa:*thing\n') + + self.cmd('create', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2', 'file3', 'file333']) + shutil.rmtree('output/input') + + # Exclude using regular expression + with open(self.exclude_file_path, 'wb') as fd: + fd.write(b're:file3+\n') + + with changedir('output'): + self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file1', 'file2']) + shutil.rmtree('output/input') + + # Mixed exclude pattern styles + with open(self.exclude_file_path, 'wb') as fd: + fd.write(b're:file(\\d)\\1\\1$\n') + fd.write(b'fm:nothingwillmatchthis\n') + fd.write(b'*/file1\n') + fd.write(b're:file2$\n') + + with changedir('output'): + self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file3']) + def test_exclude_caches(self): self.cmd('init', self.repository_location) self.create_regular_file('file1', size=1024 * 80) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index e766ed44..a61bdd28 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -10,9 +10,9 @@ import msgpack import msgpack.fallback from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, \ + prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, ExcludeRegex, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes + ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern from . import BaseTestCase, environment_variable, FakeInputs @@ -160,6 +160,15 @@ class FormatTimedeltaTestCase(BaseTestCase): ) +def check_patterns(files, paths, excludes, expected): + """Utility for testing exclusion patterns. + """ + patterns = adjust_patterns(paths, excludes) + included = [path for path in files if not exclude_path(path, patterns)] + + assert included == (files if expected is None else expected) + + @pytest.mark.parametrize("paths, excludes, expected", [ # "None" means all files, i.e. none excluded ([], [], None), @@ -184,10 +193,44 @@ def test_patterns(paths, excludes, expected): '/var/log/messages', '/var/log/dmesg', ] - patterns = adjust_patterns(paths, [ExcludePattern(p) for p in excludes]) - included = [path for path in files if not exclude_path(path, patterns)] + check_patterns(files, paths, [ExcludePattern(p) for p in excludes], expected) - assert included == (files if expected is None else expected) + +@pytest.mark.parametrize("paths, excludes, expected", [ + # "None" means all files, i.e. none excluded + ([], [], None), + (['/'], [], None), + (['/'], ['.*'], []), + (['/'], ['^/'], []), + (['/'], ['^abc$'], None), + (['/'], ['^(?!/home/)'], + ['/home/user/.profile', '/home/user/.bashrc', '/home/user2/.profile', + '/home/user2/public_html/index.html']), + ]) +def test_patterns_regex(paths, excludes, expected): + files = [ + '/srv/data', '/foo/bar', '/home', + '/home/user/.profile', '/home/user/.bashrc', + '/home/user2/.profile', '/home/user2/public_html/index.html', + '/opt/log/messages.txt', '/opt/log/dmesg.txt', + ] + + patterns = [] + + for i in excludes: + pat = ExcludeRegex(i) + assert str(pat) == i + assert pat.pattern == i + patterns.append(pat) + + check_patterns(files, paths, patterns, expected) + + +def test_regex_pattern(): + # The forward slash must match the platform-specific path separator + assert ExcludeRegex("^/$").match("/") + assert ExcludeRegex("^/$").match(os.path.sep) + assert not ExcludeRegex(r"^\\$").match("/") @pytest.mark.skipif(sys.platform in ('darwin',), reason='all but OS X test') @@ -196,31 +239,40 @@ class PatternNonAsciiTestCase(BaseTestCase): pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert not i.match("ba\N{COMBINING ACUTE ACCENT}/foo") assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert not e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert not er.match("ba\N{COMBINING ACUTE ACCENT}/foo") def testDecomposedUnicode(self): pattern = 'ba\N{COMBINING ACUTE ACCENT}' i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert not i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") assert not e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert not er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") def testInvalidUnicode(self): pattern = str(b'ba\x80', 'latin1') i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert not i.match("ba/foo") assert i.match(str(b"ba\x80/foo", 'latin1')) assert not e.match("ba/foo") assert e.match(str(b"ba\x80/foo", 'latin1')) + assert not er.match("ba/foo") + assert er.match(str(b"ba\x80/foo", 'latin1')) @pytest.mark.skipif(sys.platform not in ('darwin',), reason='OS X test') @@ -229,31 +281,40 @@ class OSXPatternNormalizationTestCase(BaseTestCase): pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") def testDecomposedUnicode(self): pattern = 'ba\N{COMBINING ACUTE ACCENT}' i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") + assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") def testInvalidUnicode(self): pattern = str(b'ba\x80', 'latin1') i = IncludePattern(pattern) e = ExcludePattern(pattern) + er = ExcludeRegex("^{}/foo$".format(pattern)) assert not i.match("ba/foo") assert i.match(str(b"ba\x80/foo", 'latin1')) assert not e.match("ba/foo") assert e.match(str(b"ba\x80/foo", 'latin1')) + assert not er.match("ba/foo") + assert er.match(str(b"ba\x80/foo", 'latin1')) @pytest.mark.parametrize("lines, expected", [ @@ -271,6 +332,17 @@ class OSXPatternNormalizationTestCase(BaseTestCase): "", "# EOF"], ["/more/data", "/home"]), + (["re:.*"], []), + (["re:\s"], ["/data/something00.txt", "/more/data", "/home"]), + ([r"re:(.)(\1)"], ["/more/data", "/home", "/whitespace/at/end of filename \t "]), + (["", "", "", + "# This is a test with mixed pattern styles", + # Case-insensitive pattern + "re:(?i)BAR|ME$", + "", + "*whitespace*", + "fm:*/something00*"], + ["/more/data"]), ]) def test_patterns_from_file(tmpdir, lines, expected): files = [ @@ -291,6 +363,35 @@ def test_patterns_from_file(tmpdir, lines, expected): assert evaluate(str(exclfile)) == (files if expected is None else expected) +@pytest.mark.parametrize("pattern, cls", [ + ("", ExcludePattern), + + # Default style + ("*", ExcludePattern), + ("/data/*", ExcludePattern), + + # fnmatch style + ("fm:", ExcludePattern), + ("fm:*", ExcludePattern), + ("fm:/data/*", ExcludePattern), + ("fm:fm:/data/*", ExcludePattern), + + # Regular expression + ("re:", ExcludeRegex), + ("re:.*", ExcludeRegex), + ("re:^/something/", ExcludeRegex), + ("re:re:^/something/", ExcludeRegex), + ]) +def test_parse_pattern(pattern, cls): + assert isinstance(parse_pattern(pattern), cls) + + +@pytest.mark.parametrize("pattern", ["aa:", "fo:*", "00:", "x1:abc"]) +def test_parse_pattern_error(pattern): + with pytest.raises(ValueError): + parse_pattern(pattern) + + def test_compression_specs(): with pytest.raises(ValueError): CompressionSpec('') diff --git a/docs/usage.rst b/docs/usage.rst index a3f4bfa2..891aed17 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -231,6 +231,11 @@ Examples ~/src \ --exclude '*.pyc' + # Backup home directories excluding image thumbnails (i.e. only + # /home/*/.thumbnails is excluded, not /home/*/*/.thumbnails) + $ borg create /mnt/backup::my-files /home \ + --exclude 're:^/home/[^/]+/\.thumbnails/' + # Backup the root filesystem into an archive named "root-YYYY-MM-DD" # use zlib compression (good, but slow) - default is no compression NAME="root-`date +%Y-%m-%d`" From 2369b8a0f2abe67e07c8059fa800753c04619ef7 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Wed, 13 Jan 2016 14:30:54 +0100 Subject: [PATCH 193/321] Strip whitespace when loading exclusions from file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Patterns to exclude files can be loaded from a text file using the “--exclude-from” option. Whitespace at the beginning or end of lines was not stripped. Indented comments would be interpreted as a pattern and a misplaced space at the end of a line--some text editors don't strip them--could cause an exclusion pattern to not match as desired. With the recent addition of regular expression support for exclusions the spaces can be matched if necessary (“^\s” or “\s$”), though it's highly unlikely that there are many paths deliberately starting or ending with whitespace. --- borg/archiver.py | 8 +++++--- borg/helpers.py | 6 +++--- borg/testsuite/helpers.py | 12 +++++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index beee1605..17d19824 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -632,9 +632,11 @@ class Archiver: expansion. The `--exclude-from` option permits loading exclusion patterns from a text - file with one pattern per line. Empty lines as well as lines starting with - the number sign ('#') are ignored. The optional style selector prefix is - also supported for patterns loaded from a file. + file with one pattern per line. Lines empty or starting with the number sign + ('#') after removing whitespace on both ends are ignored. The optional style + selector prefix is also supported for patterns loaded from a file. Due to + whitespace removal paths with whitespace at the beginning or end can only be + excluded using regular expressions. Examples: diff --git a/borg/helpers.py b/borg/helpers.py index 23e506f2..25fa0868 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -236,10 +236,10 @@ def parse_timestamp(timestamp): def load_excludes(fh): - """Load and parse exclude patterns from file object. Empty lines and lines starting with '#' are ignored, but - whitespace is not stripped. + """Load and parse exclude patterns from file object. Lines empty or starting with '#' after stripping whitespace on + both line ends are ignored. """ - patterns = (line.rstrip('\r\n') for line in fh if not line.startswith('#')) + patterns = (line for line in (i.strip() for i in fh) if not line.startswith('#')) return [parse_pattern(pattern) for pattern in patterns if pattern] diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index a61bdd28..24ea572e 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -324,17 +324,16 @@ class OSXPatternNormalizationTestCase(BaseTestCase): (["*"], []), (["# Comment", "*/something00.txt", - " whitespace\t", - "/whitespace/at/end of filename \t ", + " *whitespace* ", # Whitespace before comment " #/ws*", # Empty line "", "# EOF"], - ["/more/data", "/home"]), + ["/more/data", "/home", " #/wsfoobar"]), (["re:.*"], []), (["re:\s"], ["/data/something00.txt", "/more/data", "/home"]), - ([r"re:(.)(\1)"], ["/more/data", "/home", "/whitespace/at/end of filename \t "]), + ([r"re:(.)(\1)"], ["/more/data", "/home", "\tstart/whitespace", "/whitespace/end\t"]), (["", "", "", "# This is a test with mixed pattern styles", # Case-insensitive pattern @@ -343,12 +342,15 @@ class OSXPatternNormalizationTestCase(BaseTestCase): "*whitespace*", "fm:*/something00*"], ["/more/data"]), + ([r" re:^\s "], ["/data/something00.txt", "/more/data", "/home", "/whitespace/end\t"]), + ([r" re:\s$ "], ["/data/something00.txt", "/more/data", "/home", " #/wsfoobar", "\tstart/whitespace"]), ]) def test_patterns_from_file(tmpdir, lines, expected): files = [ '/data/something00.txt', '/more/data', '/home', ' #/wsfoobar', - '/whitespace/at/end of filename \t ', + '\tstart/whitespace', + '/whitespace/end\t', ] def evaluate(filename): From a7c1419b6e30fe27d945ef0a0fa565c00129e9d9 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Wed, 6 Jan 2016 22:47:34 +0100 Subject: [PATCH 194/321] docs/deployment: Add borg storage server setup example --- docs/deployment.rst | 160 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + 2 files changed, 161 insertions(+) create mode 100644 docs/deployment.rst diff --git a/docs/deployment.rst b/docs/deployment.rst new file mode 100644 index 00000000..0cd10d97 --- /dev/null +++ b/docs/deployment.rst @@ -0,0 +1,160 @@ +.. include:: global.rst.inc +.. _deployment: + +Deployment +========== + +This chapter will give an example how to setup a borg repository server for multiple +clients. + +Machines +-------- + +There are multiple machines used in this chapter and will further be named by their +respective fully qualified domain name (fqdn). + +* The backup server: `backup01.srv.local` +* The clients: + * John Doe's desktop: `johndoe.clnt.local` + * Webserver 01: `web01.srv.local` + * Application server 01: `app01.srv.local` + +User and group +-------------- + +The repository server needs to have only one UNIX user for all the clients. +Recommended user and group with additional settings: + +* User: `backup` +* Group: `backup` +* Shell: `/bin/bash` (or other capable to run the `borg serve` command) +* Home: `/home/backup` + +Most clients shall initiate a backup from the root user to catch all +users, groups and permissions (e.g. when backing up `/home`). + +Folders +------- + +The following folder tree layout is suggested on the repository server: + +* User home directory, /home/backup +* Repositories path (storage pool): /home/backup/repos +* Clients restricted paths: `/home/backup/repos/` + * johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local` + * web01.srv.local: `/home/backup/repos/web01.srv.local` + * app01.srv.local: `/home/backup/repos/app01.srv.local` + +Restrictions +------------ + +Borg is instructed to restrict clients into their own paths: +``borg serve --restrict-path /home/backup/repos/`` + +There is only one ssh key per client allowed. Keys are added for ``johndoe.clnt.local``, ``web01.srv.local`` and +``app01.srv.local``. But they will access the backup under only one UNIX user account as: +``backup@backup01.srv.local``. Every key in ``$HOME/.ssh/authorized_keys`` has a +forced command and restrictions applied as shown below: + +:: + + command="cd /home/backup/repos/; + borg serve --restrict-path /home/backup/repos/", + no-port-forwarding,no-X11-forwarding,no-pty + +**NOTE** The text shown above needs to be written on a single line! + +The options which are added to the key will perform the following: + +1. Force command on the ssh key and dont allow any other command to run +2. Change working directory +3. Run ``borg serve`` restricted at the client base path +4. Restrict ssh and do not allow stuff which imposes a security risk + +Due to the cd command we use, the server automatically changes the current working +directory so the client will not need to append the hostname to the remote URI. + +**NOTE** The setup above ignores all client given commandline parameters which are +normally appended to the `borg serve` command. + +Client +------ + +The client needs to initialize the `pictures` repository like this: + +`borg init backup@backup01.srv.local:pictures` + +Or with the full path (should actually never be used, as only for demonstrational purposes). +The server should automatically change the current working directory to the `` folder. + +`borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures` + +When `johndoe.clnt.local` tries to access a not restricted path the following error is raised. +John Doe tries to backup into the Web 01 path: `borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures` + +:: + + ~~~ SNIP ~~~ + Remote: borg.remote.PathNotAllowed: /home/backup/repos/web01.srv.local/pictures + ~~~ SNIP ~~~ + Repository path not allowed + +Ansible +------- + +Ansible takes care of all the system-specific commands to add the user, create the +folder. Even when the configuration is changed the repository server configuration is +satisfied and reproducable. + +Automate setting up an repository server with the user, group, folders and +permissions a Ansible playbook could be used. Keep in mind the playbook +uses the Arch Linux `pacman`_ +package manager to install and keep borg up-to-date. + +:: + + - hosts: backup01.srv.local + vars: + user: backup + group: backup + home: /home/backup + pool: "{{ home }}/repos" + auth_users: + - host: johndoe.clnt.local + key: "{{ lookup('file', '/path/to/keys/johndoe.clnt.local.pub') }}" + - host: web01.clnt.local + key: "{{ lookup('file', '/path/to/keys/web01.clnt.local.pub') }}" + - host: app01.clnt.local + key: "{{ lookup('file', '/path/to/keys/app01.clnt.local.pub') }}" + tasks: + - pacman: name=borg state=latest update_cache=yes + - group: name="{{ group }}" state=present + - user: name="{{ user }}" shell=/bin/bash home="{{ home }}" createhome=yes group="{{ group }}" groups= state=present + - file: path="{{ home }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory + - file: path="{{ home }}/.ssh" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory + - file: path="{{ pool }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory + - authorized_key: user="{{ user }}" + key="{{ item.key }}" + key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty' + with_items: auth_users + - file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file + - file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory + with_items: auth_users + +Enhancements +------------ + +As this chapter only describes a simple and effective setup it could be further +enhanced when supporting (a limited set) of client supplied commands. A wrapper +for starting `borg serve` could be written. Or borg itself could be enhanced to +autodetect it runs under SSH by checking the `SSH_ORIGINAL_COMMAND` environment +variable. This is left open for future improvements. + +When extending ssh autodetection in borg no external wrapper script is necessary +and no other interpreter or apllication has to be deployed. + +See also +-------- + +* `SSH Daemon manpage`_ +* `Ansible`_ diff --git a/docs/index.rst b/docs/index.rst index c6706685..89a907de 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -12,6 +12,7 @@ Borg Documentation installation quickstart usage + deployment faq support resources From 3e434ce6fb36f2d5acc45c122a229b2625858956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 13 Jan 2016 15:17:54 -0500 Subject: [PATCH 195/321] mention debian testing, ubuntu backport --- docs/installation.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 7a007e31..8e1b78fb 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -33,14 +33,15 @@ yet. Distribution Source Command ============ ===================== ======= Arch Linux `[community]`_ ``pacman -S borg`` -Debian `unstable/sid`_ ``apt install borgbackup`` -Ubuntu `Xenial Xerus 16.04`_ ``apt install borgbackup`` +Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` +Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` OS X `Brew cask`_ ``brew cask install borgbackup`` ============ ===================== ======= .. _[community]: https://www.archlinux.org/packages/?name=borg .. _unstable/sid: https://packages.debian.org/sid/borgbackup -.. _Xenial Xerus 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup +.. _Xenial 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup +.. _Wily 15.10 (backport PPA): https://launchpad.net/~neoatnhng/+archive/ubuntu/ppa .. _Brew cask: http://caskroom.io/ Please ask package maintainers to build a package or, if you can package / From 178b9dc151f3de60888610f7524c94b1ac386c5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 13 Jan 2016 15:18:03 -0500 Subject: [PATCH 196/321] sort OS list alphabetically --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 8e1b78fb..61ad8d09 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -34,8 +34,8 @@ Distribution Source Command ============ ===================== ======= Arch Linux `[community]`_ ``pacman -S borg`` Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` -Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` OS X `Brew cask`_ ``brew cask install borgbackup`` +Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` ============ ===================== ======= .. _[community]: https://www.archlinux.org/packages/?name=borg From cd14b766cabf16cc569bcf1e5fa2666cc9519204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 13 Jan 2016 15:18:19 -0500 Subject: [PATCH 197/321] add NixOS --- docs/installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 61ad8d09..186f46ea 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -34,6 +34,7 @@ Distribution Source Command ============ ===================== ======= Arch Linux `[community]`_ ``pacman -S borg`` Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` +NixOS `.nix file`_ OS X `Brew cask`_ ``brew cask install borgbackup`` Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` ============ ===================== ======= @@ -42,6 +43,7 @@ Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgba .. _unstable/sid: https://packages.debian.org/sid/borgbackup .. _Xenial 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup .. _Wily 15.10 (backport PPA): https://launchpad.net/~neoatnhng/+archive/ubuntu/ppa +.. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borg/default.nix .. _Brew cask: http://caskroom.io/ Please ask package maintainers to build a package or, if you can package / From 77238d175c2064b2d81920064fe05070751973c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 13 Jan 2016 15:23:34 -0500 Subject: [PATCH 198/321] fix table syntax and links --- docs/installation.rst | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 186f46ea..e991cc02 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -29,19 +29,20 @@ package which can be installed with the package manager. As |project_name| is still a young project, such a package might be not available for your system yet. -============ ===================== ======= -Distribution Source Command -============ ===================== ======= -Arch Linux `[community]`_ ``pacman -S borg`` -Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` -NixOS `.nix file`_ -OS X `Brew cask`_ ``brew cask install borgbackup`` -Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` -============ ===================== ======= +============ ============================================= ======= +Distribution Source Command +============ ============================================= ======= +Arch Linux `[community]`_ ``pacman -S borg`` +Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` +NixOS `.nix file`_ N/A +OS X `Brew cask`_ ``brew cask install borgbackup`` +Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` +============ ============================================= ======= .. _[community]: https://www.archlinux.org/packages/?name=borg +.. _stretch: https://packages.debian.org/stretch/borgbackup .. _unstable/sid: https://packages.debian.org/sid/borgbackup -.. _Xenial 15.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup +.. _Xenial 16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup .. _Wily 15.10 (backport PPA): https://launchpad.net/~neoatnhng/+archive/ubuntu/ppa .. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borg/default.nix .. _Brew cask: http://caskroom.io/ From d88df3edc645d9048ec4647d5c889d2bc2a7e335 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 9 Jan 2016 16:07:54 +0100 Subject: [PATCH 199/321] hashtable size follows a growth policy, fixes #527 also: refactor / dedupe some code into functions --- borg/_hashindex.c | 76 +++++++++++++++++++++++++++++---- borg/hash_sizes.py | 103 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+), 8 deletions(-) create mode 100644 borg/hash_sizes.py diff --git a/borg/_hashindex.c b/borg/_hashindex.c index 16adbdfc..9fb7266e 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -40,13 +40,26 @@ typedef struct { int upper_limit; } HashIndex; +/* prime (or w/ big prime factors) hash table sizes - otherwise performance breaks down! */ +static int hash_sizes[] = { + 1031, 2053, 4099, 8209, 16411, 32771, 65537, 131101, 262147, 445649, + 757607, 1287917, 2189459, 3065243, 4291319, 6007867, 8410991, + 11775359, 16485527, 23079703, 27695653, 33234787, 39881729, 47858071, + 57429683, 68915617, 82698751, 99238507, 119086189, 144378011, 157223263, + 173476439, 190253911, 209915011, 230493629, 253169431, 278728861, + 306647623, 337318939, 370742809, 408229973, 449387209, 493428073, + 543105119, 596976533, 657794869, 722676499, 795815791, 874066969, + 962279771, 1057701643, 1164002657, 1280003147, 1407800297, 1548442699, + 1703765389, 1873768367, 2062383853, /* 32bit int ends about here */ +}; + #define EMPTY _htole32(0xffffffff) #define DELETED _htole32(0xfffffffe) #define MAX_BUCKET_SIZE 512 #define BUCKET_LOWER_LIMIT .25 #define BUCKET_UPPER_LIMIT .75 /* don't go higher than 0.75, otherwise performance severely suffers! */ -#define MIN_BUCKETS 1031 /* must be prime, otherwise performance breaks down! */ #define MAX(x, y) ((x) > (y) ? (x): (y)) +#define NELEMS(x) (sizeof(x) / sizeof((x)[0])) #define BUCKET_ADDR(index, idx) (index->buckets + (idx * index->bucket_size)) #define BUCKET_IS_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) == DELETED) @@ -207,8 +220,8 @@ hashindex_read(const char *path) index->key_size = header.key_size; index->value_size = header.value_size; index->bucket_size = index->key_size + index->value_size; - index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0; - index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT); + index->lower_limit = get_lower_limit(index->num_buckets); + index->upper_limit = get_upper_limit(index->num_buckets); fail: if(fclose(fd) < 0) { EPRINTF_PATH(path, "fclose failed"); @@ -216,12 +229,59 @@ fail: return index; } +int get_lower_limit(int num_buckets){ + int min_buckets = hash_sizes[0]; + if (num_buckets <= min_buckets) + return 0; + return (int)(num_buckets * BUCKET_LOWER_LIMIT); +} + +int get_upper_limit(int num_buckets){ + int max_buckets = hash_sizes[NELEMS(hash_sizes) - 1]; + if (num_buckets >= max_buckets) + return max_buckets; + return (int)(num_buckets * BUCKET_UPPER_LIMIT); +} + +int size_idx(int size){ + /* find the hash_sizes index with entry >= size */ + int elems = NELEMS(hash_sizes); + int entry, i=0; + do{ + entry = hash_sizes[i++]; + }while((entry < size) && (i < elems)); + if (i >= elems) + return elems - 1; + i--; + return i; +} + +int fit_size(int current){ + int i = size_idx(current); + return hash_sizes[i]; +} + +int grow_size(int current){ + int i = size_idx(current) + 1; + int elems = NELEMS(hash_sizes); + if (i >= elems) + return hash_sizes[elems - 1]; + return hash_sizes[i]; +} + +int shrink_size(int current){ + int i = size_idx(current) - 1; + if (i < 0) + return hash_sizes[0]; + return hash_sizes[i]; +} + static HashIndex * hashindex_init(int capacity, int key_size, int value_size) { HashIndex *index; int i; - capacity = MAX(MIN_BUCKETS, capacity); + capacity = fit_size(capacity); if(!(index = malloc(sizeof(HashIndex)))) { EPRINTF("malloc header failed"); @@ -237,8 +297,8 @@ hashindex_init(int capacity, int key_size, int value_size) index->value_size = value_size; index->num_buckets = capacity; index->bucket_size = index->key_size + index->value_size; - index->lower_limit = index->num_buckets > MIN_BUCKETS ? ((int)(index->num_buckets * BUCKET_LOWER_LIMIT)) : 0; - index->upper_limit = (int)(index->num_buckets * BUCKET_UPPER_LIMIT); + index->lower_limit = get_lower_limit(index->num_buckets); + index->upper_limit = get_upper_limit(index->num_buckets); for(i = 0; i < capacity; i++) { BUCKET_MARK_EMPTY(index, i); } @@ -302,7 +362,7 @@ hashindex_set(HashIndex *index, const void *key, const void *value) if(idx < 0) { if(index->num_entries > index->upper_limit) { - if(!hashindex_resize(index, index->num_buckets * 2)) { + if(!hashindex_resize(index, grow_size(index->num_buckets))) { return 0; } } @@ -332,7 +392,7 @@ hashindex_delete(HashIndex *index, const void *key) BUCKET_MARK_DELETED(index, idx); index->num_entries -= 1; if(index->num_entries < index->lower_limit) { - if(!hashindex_resize(index, index->num_buckets / 2)) { + if(!hashindex_resize(index, shrink_size(index->num_buckets))) { return 0; } } diff --git a/borg/hash_sizes.py b/borg/hash_sizes.py new file mode 100644 index 00000000..68e6e160 --- /dev/null +++ b/borg/hash_sizes.py @@ -0,0 +1,103 @@ +""" +Compute hashtable sizes with nices properties +- prime sizes (for small to medium sizes) +- 2 prime-factor sizes (for big sizes) +- fast growth for small sizes +- slow growth for big sizes + +Note: + this is just a tool for developers. + within borgbackup, it is just used to generate hash_sizes definition for _hashindex.c. +""" + +from collections import namedtuple + +K, M, G = 2**10, 2**20, 2**30 + +# hash table size (in number of buckets) +start, end_p1, end_p2 = 1 * K, 127 * M, 2 * G - 10 * M # stay well below 2^31 - 1 + +Policy = namedtuple("Policy", "upto grow") + +policies = [ + # which growth factor to use when growing a hashtable of size < upto + # grow fast (*2.0) at the start so we do not have to resize too often (expensive). + # grow slow (*1.1) for huge hash tables (do not jump too much in memory usage) + Policy(256*K, 2.0), + Policy(2*M, 1.7), + Policy(16*M, 1.4), + Policy(128*M, 1.2), + Policy(2*G-1, 1.1), +] + + +# slightly modified version of: +# http://www.macdevcenter.com/pub/a/python/excerpt/pythonckbk_chap1/index1.html?page=2 +def eratosthenes(): + """Yields the sequence of prime numbers via the Sieve of Eratosthenes.""" + D = {} # map each composite integer to its first-found prime factor + q = 2 # q gets 2, 3, 4, 5, ... ad infinitum + while True: + p = D.pop(q, None) + if p is None: + # q not a key in D, so q is prime, therefore, yield it + yield q + # mark q squared as not-prime (with q as first-found prime factor) + D[q * q] = q + else: + # let x <- smallest (N*p)+q which wasn't yet known to be composite + # we just learned x is composite, with p first-found prime factor, + # since p is the first-found prime factor of q -- find and mark it + x = p + q + while x in D: + x += p + D[x] = p + q += 1 + + +def two_prime_factors(pfix=65537): + """Yields numbers with 2 prime factors pfix and p.""" + for p in eratosthenes(): + yield pfix * p + + +def get_grow_factor(size): + for p in policies: + if size < p.upto: + return p.grow + + +def find_bigger_prime(gen, i): + while True: + p = next(gen) + if p >= i: + return p + + +def main(): + sizes = [] + i = start + + gen = eratosthenes() + while i < end_p1: + grow_factor = get_grow_factor(i) + p = find_bigger_prime(gen, i) + sizes.append(p) + i = int(i * grow_factor) + + gen = two_prime_factors() # for lower ram consumption + while i < end_p2: + grow_factor = get_grow_factor(i) + p = find_bigger_prime(gen, i) + sizes.append(p) + i = int(i * grow_factor) + + print("""\ +static int hash_sizes[] = { + %s +}; +""" % ', '.join(str(size) for size in sizes)) + + +if __name__ == '__main__': + main() From 91cde721b4158f191936aceaace13bf3080f9599 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 9 Jan 2016 16:16:41 +0100 Subject: [PATCH 200/321] hashindex: minor refactor - rename BUCKET_(LOWER|UPPER)_LIMIT to HASH_(MIN|MAX)_LOAD as this value is usually called the hash table's minimum/maximum load factor. - remove MAX_BUCKET_SIZE (not used) - regroup/reorder definitions --- borg/_hashindex.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index 9fb7266e..b4db4fda 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -53,20 +53,22 @@ static int hash_sizes[] = { 1703765389, 1873768367, 2062383853, /* 32bit int ends about here */ }; -#define EMPTY _htole32(0xffffffff) -#define DELETED _htole32(0xfffffffe) -#define MAX_BUCKET_SIZE 512 -#define BUCKET_LOWER_LIMIT .25 -#define BUCKET_UPPER_LIMIT .75 /* don't go higher than 0.75, otherwise performance severely suffers! */ +#define HASH_MIN_LOAD .25 +#define HASH_MAX_LOAD .75 /* don't go higher than 0.75, otherwise performance severely suffers! */ + #define MAX(x, y) ((x) > (y) ? (x): (y)) #define NELEMS(x) (sizeof(x) / sizeof((x)[0])) + +#define EMPTY _htole32(0xffffffff) +#define DELETED _htole32(0xfffffffe) + #define BUCKET_ADDR(index, idx) (index->buckets + (idx * index->bucket_size)) +#define BUCKET_MATCHES_KEY(index, idx, key) (memcmp(key, BUCKET_ADDR(index, idx), index->key_size) == 0) + #define BUCKET_IS_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) == DELETED) #define BUCKET_IS_EMPTY(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) == EMPTY) -#define BUCKET_MATCHES_KEY(index, idx, key) (memcmp(key, BUCKET_ADDR(index, idx), index->key_size) == 0) - #define BUCKET_MARK_DELETED(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = DELETED) #define BUCKET_MARK_EMPTY(index, idx) (*((uint32_t *)(BUCKET_ADDR(index, idx) + index->key_size)) = EMPTY) @@ -233,14 +235,14 @@ int get_lower_limit(int num_buckets){ int min_buckets = hash_sizes[0]; if (num_buckets <= min_buckets) return 0; - return (int)(num_buckets * BUCKET_LOWER_LIMIT); + return (int)(num_buckets * HASH_MIN_LOAD); } int get_upper_limit(int num_buckets){ int max_buckets = hash_sizes[NELEMS(hash_sizes) - 1]; if (num_buckets >= max_buckets) return max_buckets; - return (int)(num_buckets * BUCKET_UPPER_LIMIT); + return (int)(num_buckets * HASH_MAX_LOAD); } int size_idx(int size){ From 09665805e8b77229056ce90149b144c2e583b45c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 9 Jan 2016 17:27:45 +0100 Subject: [PATCH 201/321] move func defs to avoid implicit declaration compiler warning --- borg/_hashindex.c | 94 +++++++++++++++++++++++------------------------ 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index b4db4fda..6dde62fe 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -145,6 +145,53 @@ hashindex_resize(HashIndex *index, int capacity) return 1; } +int get_lower_limit(int num_buckets){ + int min_buckets = hash_sizes[0]; + if (num_buckets <= min_buckets) + return 0; + return (int)(num_buckets * HASH_MIN_LOAD); +} + +int get_upper_limit(int num_buckets){ + int max_buckets = hash_sizes[NELEMS(hash_sizes) - 1]; + if (num_buckets >= max_buckets) + return max_buckets; + return (int)(num_buckets * HASH_MAX_LOAD); +} + +int size_idx(int size){ + /* find the hash_sizes index with entry >= size */ + int elems = NELEMS(hash_sizes); + int entry, i=0; + do{ + entry = hash_sizes[i++]; + }while((entry < size) && (i < elems)); + if (i >= elems) + return elems - 1; + i--; + return i; +} + +int fit_size(int current){ + int i = size_idx(current); + return hash_sizes[i]; +} + +int grow_size(int current){ + int i = size_idx(current) + 1; + int elems = NELEMS(hash_sizes); + if (i >= elems) + return hash_sizes[elems - 1]; + return hash_sizes[i]; +} + +int shrink_size(int current){ + int i = size_idx(current) - 1; + if (i < 0) + return hash_sizes[0]; + return hash_sizes[i]; +} + /* Public API */ static HashIndex * hashindex_read(const char *path) @@ -231,53 +278,6 @@ fail: return index; } -int get_lower_limit(int num_buckets){ - int min_buckets = hash_sizes[0]; - if (num_buckets <= min_buckets) - return 0; - return (int)(num_buckets * HASH_MIN_LOAD); -} - -int get_upper_limit(int num_buckets){ - int max_buckets = hash_sizes[NELEMS(hash_sizes) - 1]; - if (num_buckets >= max_buckets) - return max_buckets; - return (int)(num_buckets * HASH_MAX_LOAD); -} - -int size_idx(int size){ - /* find the hash_sizes index with entry >= size */ - int elems = NELEMS(hash_sizes); - int entry, i=0; - do{ - entry = hash_sizes[i++]; - }while((entry < size) && (i < elems)); - if (i >= elems) - return elems - 1; - i--; - return i; -} - -int fit_size(int current){ - int i = size_idx(current); - return hash_sizes[i]; -} - -int grow_size(int current){ - int i = size_idx(current) + 1; - int elems = NELEMS(hash_sizes); - if (i >= elems) - return hash_sizes[elems - 1]; - return hash_sizes[i]; -} - -int shrink_size(int current){ - int i = size_idx(current) - 1; - if (i < 0) - return hash_sizes[0]; - return hash_sizes[i]; -} - static HashIndex * hashindex_init(int capacity, int key_size, int value_size) { From 083f5e31efd082aa738f374680756ba57f25a3db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Jan 2016 03:20:17 +0100 Subject: [PATCH 202/321] hashindex: fix upper limit use num_buckets (== fully use what we currently have allocated) --- borg/_hashindex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index 6dde62fe..247454d2 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -155,7 +155,7 @@ int get_lower_limit(int num_buckets){ int get_upper_limit(int num_buckets){ int max_buckets = hash_sizes[NELEMS(hash_sizes) - 1]; if (num_buckets >= max_buckets) - return max_buckets; + return num_buckets; return (int)(num_buckets * HASH_MAX_LOAD); } From 5cb47cbedda7e4800679d36ddb416b9b7e40bb51 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Jan 2016 03:56:12 +0100 Subject: [PATCH 203/321] hashindex: explain hash_sizes --- borg/_hashindex.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/borg/_hashindex.c b/borg/_hashindex.c index 247454d2..f1aa0aa8 100644 --- a/borg/_hashindex.c +++ b/borg/_hashindex.c @@ -40,7 +40,15 @@ typedef struct { int upper_limit; } HashIndex; -/* prime (or w/ big prime factors) hash table sizes - otherwise performance breaks down! */ +/* prime (or w/ big prime factors) hash table sizes + * not sure we need primes for borg's usage (as we have a hash function based + * on sha256, we can assume an even, seemingly random distribution of values), + * but OTOH primes don't harm. + * also, growth of the sizes starts with fast-growing 2x steps, but slows down + * more and more down to 1.1x. this is to avoid huge jumps in memory allocation, + * like e.g. 4G -> 8G. + * these values are generated by hash_sizes.py. + */ static int hash_sizes[] = { 1031, 2053, 4099, 8209, 16411, 32771, 65537, 131101, 262147, 445649, 757607, 1287917, 2189459, 3065243, 4291319, 6007867, 8410991, From b6d0ee7c13f1054bdbb7faa1cdf77221e122c6cb Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Thu, 14 Jan 2016 17:00:45 +0100 Subject: [PATCH 204/321] Capitalization fixes for upgrade help epilog --- borg/archiver.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index f3abe8f4..3c09840c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1112,41 +1112,41 @@ class Archiver: help='repository to prune') upgrade_epilog = textwrap.dedent(""" - upgrade an existing Borg repository. this currently - only support converting an Attic repository, but may + Upgrade an existing Borg repository. This currently + only supports converting an Attic repository, but may eventually be extended to cover major Borg upgrades as well. - it will change the magic strings in the repository's segments - to match the new Borg magic strings. the keyfiles found in + It will change the magic strings in the repository's segments + to match the new Borg magic strings. The keyfiles found in $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and copied to $BORG_KEYS_DIR or ~/.borg/keys. - the cache files are converted, from $ATTIC_CACHE_DIR or + The cache files are converted, from $ATTIC_CACHE_DIR or ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the cache layout between Borg and Attic changed, so it is possible the first backup after the conversion takes longer than expected due to the cache resync. - upgrade should be able to resume if interrupted, although it - will still iterate over all segments. if you want to start + Upgrade should be able to resume if interrupted, although it + will still iterate over all segments. If you want to start from scratch, use `borg delete` over the copied repository to make sure the cache files are also removed: borg delete borg - unless ``--inplace`` is specified, the upgrade process first + Unless ``--inplace`` is specified, the upgrade process first creates a backup copy of the repository, in - REPOSITORY.upgrade-DATETIME, using hardlinks. this takes + REPOSITORY.upgrade-DATETIME, using hardlinks. This takes longer than in place upgrades, but is much safer and gives - progress information (as opposed to ``cp -al``). once you are + progress information (as opposed to ``cp -al``). Once you are satisfied with the conversion, you can safely destroy the backup copy. - WARNING: running the upgrade in place will make the current + WARNING: Running the upgrade in place will make the current copy unusable with older version, with no way of going back - to previous versions. this can PERMANENTLY DAMAGE YOUR + to previous versions. This can PERMANENTLY DAMAGE YOUR REPOSITORY! Attic CAN NOT READ BORG REPOSITORIES, as the - magic strings have changed. you have been warned.""") + magic strings have changed. You have been warned.""") subparser = subparsers.add_parser('upgrade', parents=[common_parser], description=self.do_upgrade.__doc__, epilog=upgrade_epilog, From 96f88a29d25ef3d690ea24f3e1ced8299f12c365 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Jan 2016 18:57:05 +0100 Subject: [PATCH 205/321] add --list option for borg create like --stats enables statistics output, --list enables the file/dirs list output. --- borg/archiver.py | 6 +++++- borg/testsuite/archiver.py | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 3c09840c..7ba11a03 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -80,7 +80,7 @@ class Archiver: logger.warning(msg) def print_file_status(self, status, path): - if self.output_filter is None or status in self.output_filter: + if self.output_list and (self.output_filter is None or status in self.output_filter): logger.info("%1s %s", status, remove_surrogates(path)) def do_serve(self, args): @@ -129,6 +129,7 @@ class Archiver: def do_create(self, args): """Create new archive""" self.output_filter = args.output_filter + self.output_list = args.output_list dry_run = args.dry_run t0 = datetime.now() if not dry_run: @@ -858,6 +859,9 @@ class Archiver: help="""show progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: %(default)s""") + subparser.add_argument('--list', dest='output_list', + action='store_true', default=False, + help='output verbose list of items (files, dirs, ...)') subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', help='only display items with the given status characters') subparser.add_argument('-e', '--exclude', dest='excludes', diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 58d20c52..d264224a 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -762,11 +762,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago self.create_regular_file('file2', size=1024 * 80) self.cmd('init', self.repository_location) - output = self.cmd('create', '-v', self.repository_location + '::test', 'input') + output = self.cmd('create', '-v', '--list', self.repository_location + '::test', 'input') self.assert_in("A input/file1", output) self.assert_in("A input/file2", output) # should find first file as unmodified - output = self.cmd('create', '-v', self.repository_location + '::test1', 'input') + output = self.cmd('create', '-v', '--list', self.repository_location + '::test1', 'input') self.assert_in("U input/file1", output) # this is expected, although surprising, for why, see: # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file @@ -785,7 +785,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', self.repository_location + '::test0', 'input') self.assert_not_in('file1', output) # should list the file as unchanged - output = self.cmd('create', '-v', '--filter=U', self.repository_location + '::test1', 'input') + output = self.cmd('create', '-v', '--list', '--filter=U', self.repository_location + '::test1', 'input') self.assert_in('file1', output) # should *not* list the file as changed output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test2', 'input') @@ -793,7 +793,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): # change the file self.create_regular_file('file1', size=1024 * 100) # should list the file as changed - output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test3', 'input') + output = self.cmd('create', '-v', '--list', '--filter=AM', self.repository_location + '::test3', 'input') self.assert_in('file1', output) def test_cmdline_compatibility(self): From 8ce84cab3021aaa39b6f32a026d7e90b761d03d8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Jan 2016 19:34:07 +0100 Subject: [PATCH 206/321] update docs / make them more clear about -v --- docs/quickstart.rst | 4 ++-- docs/usage.rst | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 42201e89..296321c1 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -46,7 +46,7 @@ A step by step example 3. The next day create a new archive called *Tuesday*:: - $ borg create --stats /mnt/backup::Tuesday ~/src ~/Documents + $ borg create -v --stats /mnt/backup::Tuesday ~/src ~/Documents This backup will be a lot quicker and a lot smaller since only new never before seen data is stored. The ``--stats`` option causes |project_name| to @@ -101,7 +101,7 @@ certain number of old archives:: # Backup all of /home and /var/www except a few # excluded directories - borg create --stats \ + borg create -v --stats \ $REPOSITORY::`hostname`-`date +%Y-%m-%d` \ /home \ /var/www \ diff --git a/docs/usage.rst b/docs/usage.rst index 891aed17..a072a96c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -29,6 +29,10 @@ Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL While you can set misc. log levels, do not expect that every command will give different output on different log levels - it's just a possibility. +..warning:: While some options (like --stats or --list) will emit more +informational messages, you have to use INFO (or lower) log level to make +them show up in log output. Use `-v` or a logging configuration. + Return codes ~~~~~~~~~~~~ @@ -269,7 +273,7 @@ Examples $ borg extract /mnt/backup::my-files # Extract entire archive and list files while processing - $ borg extract -v /mnt/backup::my-files + $ borg extract -v --list /mnt/backup::my-files # Extract the "src" directory $ borg extract /mnt/backup::my-files home/USERNAME/src @@ -453,7 +457,7 @@ Here are misc. notes about topics that are maybe not covered in enough detail in Item flags ~~~~~~~~~~ -`borg create -v` outputs a verbose list of all files, directories and other +`borg create -v --list` outputs a verbose list of all files, directories and other file system items it considered (no matter whether they had content changes or not). For each item, it prefixes a single-letter flag that indicates type and/or status of the item. From 6cedfbede9bad04cd1148895844170a776a0898f Mon Sep 17 00:00:00 2001 From: Danny Edel Date: Fri, 15 Jan 2016 09:24:00 +0100 Subject: [PATCH 207/321] Correct small typos in changes and usage --- docs/changes.rst | 2 +- docs/usage.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 5c6e8121..fb958aa9 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -161,7 +161,7 @@ Other changes: - improve file size displays, more flexible size formatters - explicitly commit to the units standard, #289 -- archiver: add E status (means that an error occured when processing this +- archiver: add E status (means that an error occurred when processing this (single) item - do binary releases via "github releases", closes #214 - create: use -x and --one-file-system (was: --do-not-cross-mountpoints), #296 diff --git a/docs/usage.rst b/docs/usage.rst index a072a96c..10abe666 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -441,7 +441,7 @@ Miscellaneous Help Debug Commands -------------- -There are some more commands (all starting with "debug-") wich are are all +There are some more commands (all starting with "debug-") which are all **not intended for normal use** and **potentially very dangerous** if used incorrectly. They exist to improve debugging capabilities without direct system access, e.g. From d08c51bdfc553592a5359dc854019894ce6c1884 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 10:34:05 +0100 Subject: [PATCH 208/321] add gource video to resources docs, fixes #507 --- docs/resources.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/resources.rst b/docs/resources.rst index 88e76dac..4113c11d 100644 --- a/docs/resources.rst +++ b/docs/resources.rst @@ -30,6 +30,8 @@ Some of them refer to attic, but you can do the same stuff (and more) with borgb - "Attic Backup: Mount your encrypted backups over ssh", 2014 (video, english): `youtube `_ +- "Evolution of Borg", Oct 2015 (gource visualization of attic and borg development): + `youtube `_ Software -------- From e644dae793ea5718029bd688cd18475ec5fb1764 Mon Sep 17 00:00:00 2001 From: Gianfranco Costamagna Date: Fri, 15 Jan 2016 13:50:24 +0100 Subject: [PATCH 209/321] Move to my ppa and add Trusty/Vivid packages --- docs/installation.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index e991cc02..9ab397ee 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -37,13 +37,17 @@ Debian `stretch`_, `unstable/sid`_ ``apt install borgbac NixOS `.nix file`_ N/A OS X `Brew cask`_ ``brew cask install borgbackup`` Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` +Ubuntu `Vivid 15.04 (backport PPA)`_ ``apt install borgbackup`` +Ubuntu `Trusty 14.04 (backport PPA)`_ ``apt install borgbackup`` ============ ============================================= ======= .. _[community]: https://www.archlinux.org/packages/?name=borg .. _stretch: https://packages.debian.org/stretch/borgbackup .. _unstable/sid: https://packages.debian.org/sid/borgbackup .. _Xenial 16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup -.. _Wily 15.10 (backport PPA): https://launchpad.net/~neoatnhng/+archive/ubuntu/ppa +.. _Wily 15.10 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup +.. _Vivid 15.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup +.. _Trusty 14.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup .. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borg/default.nix .. _Brew cask: http://caskroom.io/ From 2c7ab8595dc69b092d6922ce5a54c8adccdaab53 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Fri, 15 Jan 2016 16:52:55 +0100 Subject: [PATCH 210/321] Refactor Unicode pattern tests The unit tests for Unicode in path patterns contained a lot of unnecessary duplication. One set of duplication was for Mac OS X (also known as Darwin) as it normalizes Unicode in paths to NFD. Then each test case was repeated for every type of pattern. With this change the tests become parametrized using py.test. The duplicated code has been removed. --- borg/testsuite/helpers.py | 95 +++++++++------------------------------ 1 file changed, 20 insertions(+), 75 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 24ea572e..b919cda3 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -233,88 +233,33 @@ def test_regex_pattern(): assert not ExcludeRegex(r"^\\$").match("/") -@pytest.mark.skipif(sys.platform in ('darwin',), reason='all but OS X test') -class PatternNonAsciiTestCase(BaseTestCase): - def testComposedUnicode(self): - pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) - - assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert not i.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert not e.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert not er.match("ba\N{COMBINING ACUTE ACCENT}/foo") - - def testDecomposedUnicode(self): - pattern = 'ba\N{COMBINING ACUTE ACCENT}' - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) - - assert not i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert not e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert not er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") - - def testInvalidUnicode(self): - pattern = str(b'ba\x80', 'latin1') - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) - - assert not i.match("ba/foo") - assert i.match(str(b"ba\x80/foo", 'latin1')) - assert not e.match("ba/foo") - assert e.match(str(b"ba\x80/foo", 'latin1')) - assert not er.match("ba/foo") - assert er.match(str(b"ba\x80/foo", 'latin1')) +def use_normalized_unicode(): + return sys.platform in ("darwin",) -@pytest.mark.skipif(sys.platform not in ('darwin',), reason='OS X test') -class OSXPatternNormalizationTestCase(BaseTestCase): - def testComposedUnicode(self): - pattern = 'b\N{LATIN SMALL LETTER A WITH ACUTE}' - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) +def _make_test_patterns(pattern): + return [IncludePattern(pattern), + ExcludePattern(pattern), + ExcludeRegex("^{}/foo$".format(pattern)), + ] - assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") - def testDecomposedUnicode(self): - pattern = 'ba\N{COMBINING ACUTE ACCENT}' - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) +@pytest.mark.parametrize("pattern", _make_test_patterns("b\N{LATIN SMALL LETTER A WITH ACUTE}")) +def test_composed_unicode_pattern(pattern): + assert pattern.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") + assert pattern.match("ba\N{COMBINING ACUTE ACCENT}/foo") == use_normalized_unicode() - assert i.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert i.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert e.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert e.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert er.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") - assert er.match("ba\N{COMBINING ACUTE ACCENT}/foo") - def testInvalidUnicode(self): - pattern = str(b'ba\x80', 'latin1') - i = IncludePattern(pattern) - e = ExcludePattern(pattern) - er = ExcludeRegex("^{}/foo$".format(pattern)) +@pytest.mark.parametrize("pattern", _make_test_patterns("ba\N{COMBINING ACUTE ACCENT}")) +def test_decomposed_unicode_pattern(pattern): + assert pattern.match("b\N{LATIN SMALL LETTER A WITH ACUTE}/foo") == use_normalized_unicode() + assert pattern.match("ba\N{COMBINING ACUTE ACCENT}/foo") - assert not i.match("ba/foo") - assert i.match(str(b"ba\x80/foo", 'latin1')) - assert not e.match("ba/foo") - assert e.match(str(b"ba\x80/foo", 'latin1')) - assert not er.match("ba/foo") - assert er.match(str(b"ba\x80/foo", 'latin1')) + +@pytest.mark.parametrize("pattern", _make_test_patterns(str(b"ba\x80", "latin1"))) +def test_invalid_unicode_pattern(pattern): + assert not pattern.match("ba/foo") + assert pattern.match(str(b"ba\x80/foo", "latin1")) @pytest.mark.parametrize("lines, expected", [ From 3a39ddbd83a77b6ed4b3c80b3deb546f2f8efb67 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Fri, 15 Jan 2016 15:41:02 +0100 Subject: [PATCH 211/321] Rename pattern classes for consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The class names “IncludePattern” and “ExcludePattern” may have been appropriate when they were the only styles. With the recent addition of regular expression support and with at least one more style being added in forthcoming changes these classes should be renamed to be more descriptive. “ExcludeRegex” is also renamed to match the new names. --- borg/archiver.py | 4 ++-- borg/helpers.py | 16 +++++++-------- borg/testsuite/helpers.py | 42 +++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 7ba11a03..588fe395 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,7 +17,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, parse_pattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ + format_file_mode, parse_pattern, PathPrefixPattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ @@ -314,7 +314,7 @@ class Archiver: while dirs: archive.extract_item(dirs.pop(-1)) for pattern in (patterns or []): - if isinstance(pattern, IncludePattern) and pattern.match_count == 0: + if isinstance(pattern, PathPrefixPattern) and pattern.match_count == 0: self.print_warning("Include pattern '%s' never matched.", pattern) return self.exit_code diff --git a/borg/helpers.py b/borg/helpers.py index 4647d78d..ce344e3f 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -259,7 +259,7 @@ def update_excludes(args): def adjust_patterns(paths, excludes): if paths: - return (excludes or []) + [IncludePattern(path) for path in paths] + [ExcludePattern('*')] + return (excludes or []) + [PathPrefixPattern(path) for path in paths] + [FnmatchPattern('*')] else: return excludes @@ -270,7 +270,7 @@ def exclude_path(path, patterns): """ for pattern in (patterns or []): if pattern.match(path): - return isinstance(pattern, (ExcludePattern, ExcludeRegex)) + return isinstance(pattern, (FnmatchPattern, RegexPattern)) return False @@ -326,14 +326,14 @@ class PatternBase: raise NotImplementedError -# For both IncludePattern and ExcludePattern, we require that +# For both PathPrefixPattern and FnmatchPattern, we require that # the pattern either match the whole path or an initial segment # of the path up to but not including a path separator. To # unify the two cases, we add a path separator to the end of # the path before matching. -class IncludePattern(PatternBase): +class PathPrefixPattern(PatternBase): """Literal files or directories listed on the command line for some operations (e.g. extract, but not create). If a directory is specified, all paths that start with that @@ -346,7 +346,7 @@ class IncludePattern(PatternBase): return (path + os.path.sep).startswith(self.pattern) -class ExcludePattern(PatternBase): +class FnmatchPattern(PatternBase): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ @@ -366,7 +366,7 @@ class ExcludePattern(PatternBase): return (self.regex.match(path + os.path.sep) is not None) -class ExcludeRegex(PatternBase): +class RegexPattern(PatternBase): """Regular expression to exclude. """ def _prepare(self, pattern): @@ -383,8 +383,8 @@ class ExcludeRegex(PatternBase): _DEFAULT_PATTERN_STYLE = "fm" _PATTERN_STYLES = { - "fm": ExcludePattern, - "re": ExcludeRegex, + "fm": FnmatchPattern, + "re": RegexPattern, } diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index b919cda3..3da955d7 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -9,8 +9,8 @@ import sys import msgpack import msgpack.fallback -from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, ExcludeRegex, \ +from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ + prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern from . import BaseTestCase, environment_variable, FakeInputs @@ -193,7 +193,7 @@ def test_patterns(paths, excludes, expected): '/var/log/messages', '/var/log/dmesg', ] - check_patterns(files, paths, [ExcludePattern(p) for p in excludes], expected) + check_patterns(files, paths, [FnmatchPattern(p) for p in excludes], expected) @pytest.mark.parametrize("paths, excludes, expected", [ @@ -218,7 +218,7 @@ def test_patterns_regex(paths, excludes, expected): patterns = [] for i in excludes: - pat = ExcludeRegex(i) + pat = RegexPattern(i) assert str(pat) == i assert pat.pattern == i patterns.append(pat) @@ -228,9 +228,9 @@ def test_patterns_regex(paths, excludes, expected): def test_regex_pattern(): # The forward slash must match the platform-specific path separator - assert ExcludeRegex("^/$").match("/") - assert ExcludeRegex("^/$").match(os.path.sep) - assert not ExcludeRegex(r"^\\$").match("/") + assert RegexPattern("^/$").match("/") + assert RegexPattern("^/$").match(os.path.sep) + assert not RegexPattern(r"^\\$").match("/") def use_normalized_unicode(): @@ -238,9 +238,9 @@ def use_normalized_unicode(): def _make_test_patterns(pattern): - return [IncludePattern(pattern), - ExcludePattern(pattern), - ExcludeRegex("^{}/foo$".format(pattern)), + return [PathPrefixPattern(pattern), + FnmatchPattern(pattern), + RegexPattern("^{}/foo$".format(pattern)), ] @@ -311,23 +311,23 @@ def test_patterns_from_file(tmpdir, lines, expected): @pytest.mark.parametrize("pattern, cls", [ - ("", ExcludePattern), + ("", FnmatchPattern), # Default style - ("*", ExcludePattern), - ("/data/*", ExcludePattern), + ("*", FnmatchPattern), + ("/data/*", FnmatchPattern), # fnmatch style - ("fm:", ExcludePattern), - ("fm:*", ExcludePattern), - ("fm:/data/*", ExcludePattern), - ("fm:fm:/data/*", ExcludePattern), + ("fm:", FnmatchPattern), + ("fm:*", FnmatchPattern), + ("fm:/data/*", FnmatchPattern), + ("fm:fm:/data/*", FnmatchPattern), # Regular expression - ("re:", ExcludeRegex), - ("re:.*", ExcludeRegex), - ("re:^/something/", ExcludeRegex), - ("re:re:^/something/", ExcludeRegex), + ("re:", RegexPattern), + ("re:.*", RegexPattern), + ("re:^/something/", RegexPattern), + ("re:re:^/something/", RegexPattern), ]) def test_parse_pattern(pattern, cls): assert isinstance(parse_pattern(pattern), cls) From ac9d2964a0ed45d74a52a8ddf42b282dea660adf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 19:52:19 +0100 Subject: [PATCH 212/321] exclude hash_sizes.py from coverage testing this is a one-time tool for developers to generate a value table for borg. the tool is not used at borg runtime. --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 077e3e6f..e2e8fe40 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,6 +8,7 @@ omit = */borg/fuse.py */borg/support/* */borg/testsuite/* + */borg/hash_sizes.py [report] exclude_lines = From a1e1ce552c466ce822b10dbf837be897a271f41d Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Fri, 15 Jan 2016 19:06:20 +0100 Subject: [PATCH 213/321] Update README.md with doc|stable shield, minor markup fixes on docs/deployment.rst --- README.rst | 16 ++++++++++------ docs/deployment.rst | 36 ++++++++++++++++++++---------------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/README.rst b/README.rst index 2a681479..780a914e 100644 --- a/README.rst +++ b/README.rst @@ -1,12 +1,8 @@ |screencast| -.. |screencast| image:: https://asciinema.org/a/28691.png - :alt: BorgBackup Installation and Basic Usage - :target: https://asciinema.org/a/28691?autoplay=1&speed=2 - - What is BorgBackup? =================== + BorgBackup (short: Borg) is a deduplicating backup program. Optionally, it supports compression and authenticated encryption. @@ -165,7 +161,11 @@ THIS IS SOFTWARE IN DEVELOPMENT, DECIDE YOURSELF WHETHER IT FITS YOUR NEEDS. Borg is distributed under a 3-clause BSD license, see `License`_ for the complete license. -|build| |coverage| +|doc| |build| |coverage| + +.. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=stable + :alt: Documentation + :target: http://borgbackup.readthedocs.org/en/stable/ .. |build| image:: https://travis-ci.org/borgbackup/borg.svg :alt: Build Status @@ -174,3 +174,7 @@ for the complete license. .. |coverage| image:: https://codecov.io/github/borgbackup/borg/coverage.svg?branch=master :alt: Test Coverage :target: https://codecov.io/github/borgbackup/borg?branch=master + +.. |screencast| image:: https://asciinema.org/a/28691.png + :alt: BorgBackup Installation and Basic Usage + :target: https://asciinema.org/a/28691?autoplay=1&speed=2 diff --git a/docs/deployment.rst b/docs/deployment.rst index 0cd10d97..f60467df 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -15,9 +15,10 @@ respective fully qualified domain name (fqdn). * The backup server: `backup01.srv.local` * The clients: - * John Doe's desktop: `johndoe.clnt.local` - * Webserver 01: `web01.srv.local` - * Application server 01: `app01.srv.local` + + - John Doe's desktop: `johndoe.clnt.local` + - Webserver 01: `web01.srv.local` + - Application server 01: `app01.srv.local` User and group -------------- @@ -40,10 +41,11 @@ The following folder tree layout is suggested on the repository server: * User home directory, /home/backup * Repositories path (storage pool): /home/backup/repos -* Clients restricted paths: `/home/backup/repos/` - * johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local` - * web01.srv.local: `/home/backup/repos/web01.srv.local` - * app01.srv.local: `/home/backup/repos/app01.srv.local` +* Clients restricted paths (`/home/backup/repos/`): + + - johndoe.clnt.local: `/home/backup/repos/johndoe.clnt.local` + - web01.srv.local: `/home/backup/repos/web01.srv.local` + - app01.srv.local: `/home/backup/repos/app01.srv.local` Restrictions ------------ @@ -62,7 +64,7 @@ forced command and restrictions applied as shown below: borg serve --restrict-path /home/backup/repos/", no-port-forwarding,no-X11-forwarding,no-pty -**NOTE** The text shown above needs to be written on a single line! +.. note:: The text shown above needs to be written on a single line! The options which are added to the key will perform the following: @@ -74,23 +76,25 @@ The options which are added to the key will perform the following: Due to the cd command we use, the server automatically changes the current working directory so the client will not need to append the hostname to the remote URI. -**NOTE** The setup above ignores all client given commandline parameters which are -normally appended to the `borg serve` command. +.. note:: The setup above ignores all client given commandline parameters + which are normally appended to the `borg serve` command. Client ------ The client needs to initialize the `pictures` repository like this: -`borg init backup@backup01.srv.local:pictures` + borg init backup@backup01.srv.local:pictures Or with the full path (should actually never be used, as only for demonstrational purposes). The server should automatically change the current working directory to the `` folder. -`borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures` + borg init backup@backup01.srv.local:/home/backup/repos/johndoe.clnt.local/pictures When `johndoe.clnt.local` tries to access a not restricted path the following error is raised. -John Doe tries to backup into the Web 01 path: `borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures` +John Doe tries to backup into the Web 01 path: + + borg init backup@backup01.srv.local:/home/backup/repos/web01.srv.local/pictures :: @@ -108,7 +112,7 @@ satisfied and reproducable. Automate setting up an repository server with the user, group, folders and permissions a Ansible playbook could be used. Keep in mind the playbook -uses the Arch Linux `pacman`_ +uses the Arch Linux `pacman `_ package manager to install and keep borg up-to-date. :: @@ -156,5 +160,5 @@ and no other interpreter or apllication has to be deployed. See also -------- -* `SSH Daemon manpage`_ -* `Ansible`_ +* `SSH Daemon manpage `_ +* `Ansible `_ From 888e078382d819647f2615b5c086f1507c0a5c9e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 20:56:21 +0100 Subject: [PATCH 214/321] use finer chunker granularity for items metadata stream, fixes #547, fixes #487 the items metadata stream is usually not that big (compared to the file content data) - it is just file and dir names and other metadata. if we use too rough granularity there (and big minimum chunk size), we usually will get no deduplication. --- borg/archive.py | 9 ++++++--- docs/internals.rst | 8 +++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 3edd4522..aba029f6 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -34,6 +34,9 @@ HASH_MASK_BITS = 16 # results in ~64kiB chunks statistically # defaults, use --chunker-params to override CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) +# chunker params for the items metadata stream, finer granularity +ITEMS_CHUNKER_PARAMS = (12, 16, 14, HASH_WINDOW_SIZE) + utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {}) has_mtime_ns = sys.version >= '3.3' @@ -75,7 +78,7 @@ class DownloadPipeline: class ChunkBuffer: BUFFER_SIZE = 1 * 1024 * 1024 - def __init__(self, key, chunker_params=CHUNKER_PARAMS): + def __init__(self, key, chunker_params=ITEMS_CHUNKER_PARAMS): self.buffer = BytesIO() self.packer = msgpack.Packer(unicode_errors='surrogateescape') self.chunks = [] @@ -110,7 +113,7 @@ class ChunkBuffer: class CacheChunkBuffer(ChunkBuffer): - def __init__(self, cache, key, stats, chunker_params=CHUNKER_PARAMS): + def __init__(self, cache, key, stats, chunker_params=ITEMS_CHUNKER_PARAMS): super().__init__(key, chunker_params) self.cache = cache self.stats = stats @@ -150,7 +153,7 @@ class Archive: self.end = end self.pipeline = DownloadPipeline(self.repository, self.key) if create: - self.items_buffer = CacheChunkBuffer(self.cache, self.key, self.stats, chunker_params) + self.items_buffer = CacheChunkBuffer(self.cache, self.key, self.stats) self.chunker = Chunker(self.key.chunk_seed, *chunker_params) if name in manifest.archives: raise self.AlreadyExists(name) diff --git a/docs/internals.rst b/docs/internals.rst index 5f3e96ec..059b9893 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -190,9 +190,11 @@ Each item represents a file, directory or other fs item and is stored as an it and it is reset every time an inode's metadata is changed. All items are serialized using msgpack and the resulting byte stream -is fed into the same chunker used for regular file data and turned -into deduplicated chunks. The reference to these chunks is then added -to the archive metadata. +is fed into the same chunker algorithm as used for regular file data +and turned into deduplicated chunks. The reference to these chunks is then added +to the archive metadata. To achieve a finer granularity on this metadata +stream, we use different chunker params for this chunker, which result in +smaller chunks. A chunk is stored as an object as well, of course. From 845d2144cb6a4e4f870f3e5af993d299e90ceffd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 22 Dec 2015 12:11:36 +0100 Subject: [PATCH 215/321] fix locking, partial fix for #502 the problem was that the borg process removed its own shared lock when upgrading it to an exclusive lock. this is fine if we get the exclusive lock, but if we don't, we must re-add our shared lock. this fixes the KeyError in locking.py:217 --- borg/locking.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borg/locking.py b/borg/locking.py index cd54ca79..af17d8bc 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -278,9 +278,11 @@ class UpgradableLock: try: if remove is not None: self._roster.modify(remove, REMOVE) - remove = None if len(self._roster.get(SHARED)) == 0: return # we are the only one and we keep the lock! + # restore the roster state as before (undo the roster change): + if remove is not None: + self._roster.modify(remove, ADD) except: # avoid orphan lock when an exception happens here, e.g. Ctrl-C! self._lock.release() From e68b800d01820c5a55efce58d8444c01d9c53c3f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 18:58:52 +0100 Subject: [PATCH 216/321] remove unused "repair" rpc method there is no such method in the code. we use "check" method to repair the repo, so maybe this was left over from a time when repair was separate from check. --- borg/remote.py | 1 - 1 file changed, 1 deletion(-) diff --git a/borg/remote.py b/borg/remote.py index 0c572cd9..391a70fa 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -49,7 +49,6 @@ class RepositoryServer: # pragma: no cover 'negotiate', 'open', 'put', - 'repair', 'rollback', 'save_key', 'load_key', From 0213164d461e58e6c86c73737339d292a344496b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 20:32:24 +0100 Subject: [PATCH 217/321] implement --progress option for borg upgrade, fixes #291 --- borg/archiver.py | 5 ++++- borg/upgrader.py | 12 +++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 588fe395..c56023d6 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -520,7 +520,7 @@ class Archiver: # XXX: should auto-detect if it is an attic repository here repo = AtticRepositoryUpgrader(args.location.path, create=False) try: - repo.upgrade(args.dry_run, inplace=args.inplace) + repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress) except NotImplementedError as e: print("warning: %s" % e) return self.exit_code @@ -1156,6 +1156,9 @@ class Archiver: epilog=upgrade_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_upgrade) + subparser.add_argument('-p', '--progress', dest='progress', + action='store_true', default=False, + help="""show progress display while upgrading the repository""") subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not change repository') diff --git a/borg/upgrader.py b/borg/upgrader.py index 3bd5400f..bb47429e 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -20,7 +20,7 @@ class AtticRepositoryUpgrader(Repository): kw['lock'] = False # do not create borg lock files (now) in attic repo super().__init__(*args, **kw) - def upgrade(self, dryrun=True, inplace=False): + def upgrade(self, dryrun=True, inplace=False, progress=False): """convert an attic repository to a borg repository those are the files that need to be upgraded here, from most @@ -54,7 +54,7 @@ class AtticRepositoryUpgrader(Repository): try: self.convert_cache(dryrun) self.convert_repo_index(dryrun=dryrun, inplace=inplace) - self.convert_segments(segments, dryrun=dryrun, inplace=inplace) + self.convert_segments(segments, dryrun=dryrun, inplace=inplace, progress=progress) self.borg_readme() finally: self.lock.release() @@ -68,7 +68,7 @@ class AtticRepositoryUpgrader(Repository): fd.write('This is a Borg repository\n') @staticmethod - def convert_segments(segments, dryrun=True, inplace=False): + def convert_segments(segments, dryrun=True, inplace=False, progress=False): """convert repository segments from attic to borg replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in @@ -80,12 +80,14 @@ class AtticRepositoryUpgrader(Repository): segment_count = len(segments) pi = ProgressIndicatorPercent(total=segment_count, msg="Converting segments %3.0f%%", same_line=True) for i, filename in enumerate(segments): - pi.show(i) + if progress: + pi.show(i) if dryrun: time.sleep(0.001) else: AtticRepositoryUpgrader.header_replace(filename, ATTIC_MAGIC, MAGIC, inplace=inplace) - pi.finish() + if progress: + pi.finish() @staticmethod def header_replace(filename, old_magic, new_magic, inplace=True): From 9198f6962cedfa7981d03e09bd654cf61ce00a1d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 20:46:49 +0100 Subject: [PATCH 218/321] implement --progress option for borg delete --- borg/archive.py | 13 +++++++++---- borg/archiver.py | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index aba029f6..0aa92b4d 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -18,7 +18,7 @@ from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \ Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ - st_atime_ns, st_ctime_ns, st_mtime_ns + st_atime_ns, st_ctime_ns, st_mtime_ns, ProgressIndicatorPercent from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex @@ -418,16 +418,21 @@ Number of files: {0.stats.nfiles}'''.format(self) self.cache.chunk_decref(self.id, self.stats) del self.manifest.archives[self.name] - def delete(self, stats): + def delete(self, stats, progress=False): unpacker = msgpack.Unpacker(use_list=False) - for items_id, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): + items_ids = self.metadata[b'items'] + pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) + for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + if progress: + pi.show(i) unpacker.feed(self.key.decrypt(items_id, data)) self.cache.chunk_decref(items_id, stats) for item in unpacker: if b'chunks' in item: for chunk_id, size, csize in item[b'chunks']: self.cache.chunk_decref(chunk_id, stats) - + if progress: + pi.finish() self.cache.chunk_decref(self.id, stats) del self.manifest.archives[self.name] diff --git a/borg/archiver.py b/borg/archiver.py index c56023d6..9cb7e3d1 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -338,7 +338,7 @@ class Archiver: if args.location.archive: archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() - archive.delete(stats) + archive.delete(stats, progress=args.progress) manifest.write() repository.commit(save_space=args.save_space) cache.commit() @@ -982,6 +982,9 @@ class Archiver: epilog=delete_epilog, formatter_class=argparse.RawDescriptionHelpFormatter) subparser.set_defaults(func=self.do_delete) + subparser.add_argument('-p', '--progress', dest='progress', + action='store_true', default=False, + help="""show progress display while deleting a single archive""") subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the deleted archive') From 2f05e4add907f76d87f93f78d0ad06383d6f7f6a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 20:57:22 +0100 Subject: [PATCH 219/321] use the usual commandline arguments order for borg prune examples, fixes #560 borg prune --- docs/usage.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 10abe666..3379462d 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -345,18 +345,18 @@ will see what it would do without it actually doing anything. # Keep 7 end of day and 4 additional end of week archives. # Do a dry-run without actually deleting anything. - $ borg prune /mnt/backup --dry-run --keep-daily=7 --keep-weekly=4 + $ borg prune --dry-run --keep-daily=7 --keep-weekly=4 /mnt/backup # Same as above but only apply to archive names starting with "foo": - $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --prefix=foo + $ borg prune --keep-daily=7 --keep-weekly=4 --prefix=foo /mnt/backup # Keep 7 end of day, 4 additional end of week archives, # and an end of month archive for every month: - $ borg prune /mnt/backup --keep-daily=7 --keep-weekly=4 --keep-monthly=-1 + $ borg prune --keep-daily=7 --keep-weekly=4 --keep-monthly=-1 /mnt/backup # Keep all backups in the last 10 days, 4 additional end of week archives, # and an end of month archive for every month: - $ borg prune /mnt/backup --keep-within=10d --keep-weekly=4 --keep-monthly=-1 + $ borg prune --keep-within=10d --keep-weekly=4 --keep-monthly=-1 /mnt/backup .. include:: usage/info.rst.inc From 4d73f3cdb90e52c0d7276414658cdd700af3eb78 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 23:42:54 +0100 Subject: [PATCH 220/321] implement and use context manager for RepositoryCache, fixes #548 --- borg/archive.py | 65 +++++++++++++++++++++++++------------------------ borg/cache.py | 12 ++++----- borg/fuse.py | 1 + borg/remote.py | 41 +++++++++++++++++++++++++------ 4 files changed, 73 insertions(+), 46 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index aba029f6..05e5479f 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -829,7 +829,6 @@ class ArchiveChecker: raise i += 1 - repository = cache_if_remote(self.repository) if archive is None: # we need last N or all archives archive_items = sorted(self.manifest.archives.items(), reverse=True, @@ -843,37 +842,39 @@ class ArchiveChecker: archive_items = [item for item in self.manifest.archives.items() if item[0] == archive] num_archives = 1 end = 1 - for i, (name, info) in enumerate(archive_items[:end]): - logger.info('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives)) - archive_id = info[b'id'] - if archive_id not in self.chunks: - logger.error('Archive metadata block is missing!') - self.error_found = True - del self.manifest.archives[name] - continue - mark_as_possibly_superseded(archive_id) - cdata = self.repository.get(archive_id) - data = self.key.decrypt(archive_id, cdata) - archive = StableDict(msgpack.unpackb(data)) - if archive[b'version'] != 1: - raise Exception('Unknown archive metadata version') - decode_dict(archive, (b'name', b'hostname', b'username', b'time')) - archive[b'cmdline'] = [arg.decode('utf-8', 'surrogateescape') for arg in archive[b'cmdline']] - items_buffer = ChunkBuffer(self.key) - items_buffer.write_chunk = add_callback - for item in robust_iterator(archive): - if b'chunks' in item: - verify_file_chunks(item) - items_buffer.add(item) - items_buffer.flush(flush=True) - for previous_item_id in archive[b'items']: - mark_as_possibly_superseded(previous_item_id) - archive[b'items'] = items_buffer.chunks - data = msgpack.packb(archive, unicode_errors='surrogateescape') - new_archive_id = self.key.id_hash(data) - cdata = self.key.encrypt(data) - add_reference(new_archive_id, len(data), len(cdata), cdata) - info[b'id'] = new_archive_id + + with cache_if_remote(self.repository) as repository: + for i, (name, info) in enumerate(archive_items[:end]): + logger.info('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives)) + archive_id = info[b'id'] + if archive_id not in self.chunks: + logger.error('Archive metadata block is missing!') + self.error_found = True + del self.manifest.archives[name] + continue + mark_as_possibly_superseded(archive_id) + cdata = self.repository.get(archive_id) + data = self.key.decrypt(archive_id, cdata) + archive = StableDict(msgpack.unpackb(data)) + if archive[b'version'] != 1: + raise Exception('Unknown archive metadata version') + decode_dict(archive, (b'name', b'hostname', b'username', b'time')) + archive[b'cmdline'] = [arg.decode('utf-8', 'surrogateescape') for arg in archive[b'cmdline']] + items_buffer = ChunkBuffer(self.key) + items_buffer.write_chunk = add_callback + for item in robust_iterator(archive): + if b'chunks' in item: + verify_file_chunks(item) + items_buffer.add(item) + items_buffer.flush(flush=True) + for previous_item_id in archive[b'items']: + mark_as_possibly_superseded(previous_item_id) + archive[b'items'] = items_buffer.chunks + data = msgpack.packb(archive, unicode_errors='surrogateescape') + new_archive_id = self.key.id_hash(data) + cdata = self.key.encrypt(data) + add_reference(new_archive_id, len(data), len(cdata), cdata) + info[b'id'] = new_archive_id def orphan_chunks_check(self): if self.check_all: diff --git a/borg/cache.py b/borg/cache.py index eaefc990..bfb817ab 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -340,12 +340,12 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" pass self.begin_txn() - repository = cache_if_remote(self.repository) - legacy_cleanup() - # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d - - # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk) - self.do_cache = os.path.isdir(archive_path) - self.chunks = create_master_idx(self.chunks) + with cache_if_remote(self.repository) as repository: + legacy_cleanup() + # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d - + # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk) + self.do_cache = os.path.isdir(archive_path) + self.chunks = create_master_idx(self.chunks) def add_chunk(self, id, data, stats): if not self.txn_active: diff --git a/borg/fuse.py b/borg/fuse.py index 448fe02a..0596ae67 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -238,3 +238,4 @@ class FuseOperations(llfuse.Operations): llfuse.main(single=True) finally: llfuse.close() + self.repository.close() diff --git a/borg/remote.py b/borg/remote.py index 391a70fa..d8092ec7 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -359,21 +359,45 @@ class RemoteRepository: self.preload_ids += ids -class RepositoryCache: +class RepositoryNoCache: + """A not caching Repository wrapper, passes through to repository. + + Just to have same API (including the context manager) as RepositoryCache. + """ + def __init__(self, repository): + self.repository = repository + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def get(self, key): + return next(self.get_many([key])) + + def get_many(self, keys): + for data in self.repository.get_many(keys): + yield data + + +class RepositoryCache(RepositoryNoCache): """A caching Repository wrapper Caches Repository GET operations using a local temporary Repository. """ def __init__(self, repository): - self.repository = repository + super().__init__(repository) tmppath = tempfile.mkdtemp(prefix='borg-tmp') self.caching_repo = Repository(tmppath, create=True, exclusive=True) - def __del__(self): - self.caching_repo.destroy() - - def get(self, key): - return next(self.get_many([key])) + def close(self): + if self.caching_repo is not None: + self.caching_repo.destroy() + self.caching_repo = None def get_many(self, keys): unknown_keys = [key for key in keys if key not in self.caching_repo] @@ -395,4 +419,5 @@ class RepositoryCache: def cache_if_remote(repository): if isinstance(repository, RemoteRepository): return RepositoryCache(repository) - return repository + else: + return RepositoryNoCache(repository) From 5ec2a3a49b798693f01de221609886eeacfe87cc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Jan 2016 00:28:54 +0100 Subject: [PATCH 221/321] use the RepositoryCache context manager also in fuse code --- borg/archiver.py | 27 ++++++++++++++------------- borg/fuse.py | 6 ++---- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 588fe395..b796665a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -30,7 +30,7 @@ from .repository import Repository from .cache import Cache from .key import key_creator from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS -from .remote import RepositoryServer, RemoteRepository +from .remote import RepositoryServer, RemoteRepository, cache_if_remote has_lchflags = hasattr(os, 'lchflags') @@ -380,18 +380,19 @@ class Archiver: repository = self.open_repository(args) try: - manifest, key = Manifest.load(repository) - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive) - else: - archive = None - operations = FuseOperations(key, repository, manifest, archive) - logger.info("Mounting filesystem") - try: - operations.mount(args.mountpoint, args.options, args.foreground) - except RuntimeError: - # Relevant error message already printed to stderr by fuse - self.exit_code = EXIT_ERROR + with cache_if_remote(repository) as cached_repo: + manifest, key = Manifest.load(repository) + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive) + else: + archive = None + operations = FuseOperations(key, repository, manifest, archive, cached_repo) + logger.info("Mounting filesystem") + try: + operations.mount(args.mountpoint, args.options, args.foreground) + except RuntimeError: + # Relevant error message already printed to stderr by fuse + self.exit_code = EXIT_ERROR finally: repository.close() return self.exit_code diff --git a/borg/fuse.py b/borg/fuse.py index 0596ae67..36f761e4 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -8,7 +8,6 @@ import tempfile import time from .archive import Archive from .helpers import daemonize -from .remote import cache_if_remote import msgpack @@ -34,11 +33,11 @@ class ItemCache: class FuseOperations(llfuse.Operations): """Export archive as a fuse filesystem """ - def __init__(self, key, repository, manifest, archive): + def __init__(self, key, repository, manifest, archive, cached_repo): super().__init__() self._inode_count = 0 self.key = key - self.repository = cache_if_remote(repository) + self.repository = cached_repo self.items = {} self.parent = {} self.contents = defaultdict(dict) @@ -238,4 +237,3 @@ class FuseOperations(llfuse.Operations): llfuse.main(single=True) finally: llfuse.close() - self.repository.close() From 22f218baeff94f3a57107e5105dda630e971e020 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Jan 2016 01:09:13 +0100 Subject: [PATCH 222/321] implement and use context manager for Cache, partial fix for #285 also: make check in Lock.close more precise, check for "is not None". note: a lot of blocks were just indented to be under the "with" statement, in one case a block had to be moved into a function. --- borg/archiver.py | 254 ++++++++++++++++++++++++----------------------- borg/cache.py | 7 +- 2 files changed, 134 insertions(+), 127 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9cb7e3d1..fa052098 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -97,7 +97,8 @@ class Archiver: manifest.key = key manifest.write() repository.commit() - Cache(repository, key, manifest, warn_if_unencrypted=False) + with Cache(repository, key, manifest, warn_if_unencrypted=False): + pass return self.exit_code def do_check(self, args): @@ -128,6 +129,59 @@ class Archiver: def do_create(self, args): """Create new archive""" + def create_inner(archive, cache): + # Add cache dir to inode_skip list + skip_inodes = set() + try: + st = os.stat(get_cache_dir()) + skip_inodes.add((st.st_ino, st.st_dev)) + except IOError: + pass + # Add local repository dir to inode_skip list + if not args.location.host: + try: + st = os.stat(args.location.path) + skip_inodes.add((st.st_ino, st.st_dev)) + except IOError: + pass + for path in args.paths: + if path == '-': # stdin + path = 'stdin' + if not dry_run: + try: + status = archive.process_stdin(path, cache) + except IOError as e: + status = 'E' + self.print_warning('%s: %s', path, e) + else: + status = '-' + self.print_file_status(status, path) + continue + path = os.path.normpath(path) + if args.one_file_system: + try: + restrict_dev = os.lstat(path).st_dev + except OSError as e: + self.print_warning('%s: %s', path, e) + continue + else: + restrict_dev = None + self._process(archive, cache, args.excludes, args.exclude_caches, args.exclude_if_present, + args.keep_tag_files, skip_inodes, path, restrict_dev, + read_special=args.read_special, dry_run=dry_run) + if not dry_run: + archive.save(timestamp=args.timestamp) + if args.progress: + archive.stats.show_progress(final=True) + if args.stats: + archive.end = datetime.now() + log_multi(DASHES, + str(archive), + DASHES, + str(archive.stats), + str(cache), + DASHES) + self.output_filter = args.output_filter self.output_list = args.output_list dry_run = args.dry_run @@ -138,64 +192,14 @@ class Archiver: compr_args = dict(buffer=COMPR_BUFFER) compr_args.update(args.compression) key.compressor = Compressor(**compr_args) - cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.location.archive, cache=cache, - create=True, checkpoint_interval=args.checkpoint_interval, - numeric_owner=args.numeric_owner, progress=args.progress, - chunker_params=args.chunker_params, start=t0) + with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache, + create=True, checkpoint_interval=args.checkpoint_interval, + numeric_owner=args.numeric_owner, progress=args.progress, + chunker_params=args.chunker_params, start=t0) + create_inner(archive, cache) else: - archive = cache = None - # Add cache dir to inode_skip list - skip_inodes = set() - try: - st = os.stat(get_cache_dir()) - skip_inodes.add((st.st_ino, st.st_dev)) - except IOError: - pass - # Add local repository dir to inode_skip list - if not args.location.host: - try: - st = os.stat(args.location.path) - skip_inodes.add((st.st_ino, st.st_dev)) - except IOError: - pass - for path in args.paths: - if path == '-': # stdin - path = 'stdin' - if not dry_run: - try: - status = archive.process_stdin(path, cache) - except IOError as e: - status = 'E' - self.print_warning('%s: %s', path, e) - else: - status = '-' - self.print_file_status(status, path) - continue - path = os.path.normpath(path) - if args.one_file_system: - try: - restrict_dev = os.lstat(path).st_dev - except OSError as e: - self.print_warning('%s: %s', path, e) - continue - else: - restrict_dev = None - self._process(archive, cache, args.excludes, args.exclude_caches, args.exclude_if_present, - args.keep_tag_files, skip_inodes, path, restrict_dev, - read_special=args.read_special, dry_run=dry_run) - if not dry_run: - archive.save(timestamp=args.timestamp) - if args.progress: - archive.stats.show_progress(final=True) - if args.stats: - archive.end = datetime.now() - log_multi(DASHES, - str(archive), - DASHES, - str(archive.stats), - str(cache), - DASHES) + create_inner(None, None) return self.exit_code def _process(self, archive, cache, excludes, exclude_caches, exclude_if_present, @@ -322,48 +326,48 @@ class Archiver: """Rename an existing archive""" repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - archive.rename(args.name) - manifest.write() - repository.commit() - cache.commit() + with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + archive.rename(args.name) + manifest.write() + repository.commit() + cache.commit() return self.exit_code def do_delete(self, args): """Delete an existing repository or archive""" repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = Statistics() - archive.delete(stats, progress=args.progress) - manifest.write() - repository.commit(save_space=args.save_space) - cache.commit() - logger.info("Archive deleted.") - if args.stats: - log_multi(DASHES, - stats.summary.format(label='Deleted data:', stats=stats), - str(cache), - DASHES) - else: - if not args.cache_only: - msg = [] - msg.append("You requested to completely DELETE the repository *including* all archives it contains:") - for archive_info in manifest.list_archive_infos(sort_by='ts'): - msg.append(format_archive(archive_info)) - msg.append("Type 'YES' if you understand this and want to continue: ") - msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): - self.exit_code = EXIT_ERROR - return self.exit_code - repository.destroy() - logger.info("Repository deleted.") - cache.destroy() - logger.info("Cache deleted.") + with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + stats = Statistics() + archive.delete(stats, progress=args.progress) + manifest.write() + repository.commit(save_space=args.save_space) + cache.commit() + logger.info("Archive deleted.") + if args.stats: + log_multi(DASHES, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES) + else: + if not args.cache_only: + msg = [] + msg.append("You requested to completely DELETE the repository *including* all archives it contains:") + for archive_info in manifest.list_archive_infos(sort_by='ts'): + msg.append(format_archive(archive_info)) + msg.append("Type 'YES' if you understand this and want to continue: ") + msg = '\n'.join(msg) + if not yes(msg, false_msg="Aborting.", default_notty=False, + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + self.exit_code = EXIT_ERROR + return self.exit_code + repository.destroy() + logger.info("Repository deleted.") + cache.destroy() + logger.info("Cache deleted.") return self.exit_code def do_mount(self, args): @@ -444,26 +448,25 @@ class Archiver: """Show archive details such as disk space used""" repository = self.open_repository(args) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = archive.calc_stats(cache) - print('Name:', archive.name) - print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) - print('Hostname:', archive.metadata[b'hostname']) - print('Username:', archive.metadata[b'username']) - print('Time: %s' % format_time(to_localtime(archive.ts))) - print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) - print('Number of files: %d' % stats.nfiles) - print() - print(str(stats)) - print(str(cache)) + with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + stats = archive.calc_stats(cache) + print('Name:', archive.name) + print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) + print('Hostname:', archive.metadata[b'hostname']) + print('Username:', archive.metadata[b'username']) + print('Time: %s' % format_time(to_localtime(archive.ts))) + print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) + print('Number of files: %d' % stats.nfiles) + print() + print(str(stats)) + print(str(cache)) return self.exit_code def do_prune(self, args): """Prune repository archives according to specified rules""" repository = self.open_repository(args, exclusive=True) manifest, key = Manifest.load(repository) - cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0 and args.within is None: self.print_error('At least one of the "within", "keep-hourly", "keep-daily", "keep-weekly", ' @@ -488,23 +491,24 @@ class Archiver: keep.sort(key=attrgetter('ts'), reverse=True) to_delete = [a for a in archives if a not in keep] stats = Statistics() - for archive in keep: - logger.info('Keeping archive: %s' % format_archive(archive)) - for archive in to_delete: - if args.dry_run: - logger.info('Would prune: %s' % format_archive(archive)) - else: - logger.info('Pruning archive: %s' % format_archive(archive)) - Archive(repository, key, manifest, archive.name, cache).delete(stats) - if to_delete and not args.dry_run: - manifest.write() - repository.commit(save_space=args.save_space) - cache.commit() - if args.stats: - log_multi(DASHES, - stats.summary.format(label='Deleted data:', stats=stats), - str(cache), - DASHES) + with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: + for archive in keep: + logger.info('Keeping archive: %s' % format_archive(archive)) + for archive in to_delete: + if args.dry_run: + logger.info('Would prune: %s' % format_archive(archive)) + else: + logger.info('Pruning archive: %s' % format_archive(archive)) + Archive(repository, key, manifest, archive.name, cache).delete(stats) + if to_delete and not args.dry_run: + manifest.write() + repository.commit(save_space=args.save_space) + cache.commit() + if args.stats: + log_multi(DASHES, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES) return self.exit_code def do_upgrade(self, args): diff --git a/borg/cache.py b/borg/cache.py index eaefc990..09e96e9e 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -78,7 +78,10 @@ class Cache: self.sync() self.commit() - def __del__(self): + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): self.close() def __str__(self): @@ -149,7 +152,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" self.rollback() def close(self): - if self.lock: + if self.lock is not None: self.lock.release() self.lock = None From 19eb8e2d3bacd5fa99b9a6c21c5d04b018cbfb8a Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 20:12:23 +0100 Subject: [PATCH 223/321] =?UTF-8?q?Use=20``--option``=20in=20RST=20to=20no?= =?UTF-8?q?t=20render=20"--"=20as=20"=E2=80=93".?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/internals.rst | 2 +- docs/quickstart.rst | 2 +- docs/usage.rst | 22 +++++++++++----------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 059b9893..468e38b7 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -220,7 +220,7 @@ for the archive, and stored encrypted in the keyfile. This is to prevent chunk size based fingerprinting attacks on your encrypted repo contents (to guess what files you have based on a specific set of chunk sizes). -For some more general usage hints see also `--chunker-params`. +For some more general usage hints see also ``--chunker-params``. Indexes / Caches diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 296321c1..18457e9c 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -17,7 +17,7 @@ a good amount of free space on the filesystem that has your backup repository If you run out of disk space, it can be hard or impossible to free space, because |project_name| needs free space to operate - even to delete backup -archives. There is a `--save-space` option for some commands, but even with +archives. There is a ``--save-space`` option for some commands, but even with that |project_name| will need free space to operate. You can use some monitoring process or just include the free space information diff --git a/docs/usage.rst b/docs/usage.rst index 3379462d..5d93be4b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -18,9 +18,9 @@ The log level of the builtin logging configuration defaults to WARNING. This is because we want |project_name| to be mostly silent and only output warnings (plus errors and critical messages). -Use --verbose or --info to set INFO (you will get informative output then +Use ``--verbose`` or ``--info`` to set INFO (you will get informative output then additionally to warnings, errors, critical messages). -Use --debug to set DEBUG to get output made for debugging. +Use ``--debug`` to set DEBUG to get output made for debugging. All log messages created with at least the set level will be output. @@ -29,9 +29,9 @@ Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL While you can set misc. log levels, do not expect that every command will give different output on different log levels - it's just a possibility. -..warning:: While some options (like --stats or --list) will emit more +..warning:: While some options (like ``--stats`` or ``--list``) will emit more informational messages, you have to use INFO (or lower) log level to make -them show up in log output. Use `-v` or a logging configuration. +them show up in log output. Use ``-v`` or a logging configuration. Return codes ~~~~~~~~~~~~ @@ -75,7 +75,7 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do BORG_RELOCATED_REPO_ACCESS_IS_OK For "Warning: The repository at location ... was previously located at ..." BORG_CHECK_I_KNOW_WHAT_I_AM_DOING - For "Warning: 'check --repair' is an experimental feature that might result in data loss." + For "Warning: 'check ``--repair``' is an experimental feature that might result in data loss." BORG_DELETE_I_KNOW_WHAT_I_AM_DOING For "You requested to completely DELETE the repository *including* all archives it contains: " @@ -334,11 +334,11 @@ Be careful, prune is potentially dangerous command, it will remove backup archives. The default of prune is to apply to **all archives in the repository** unless -you restrict its operation to a subset of the archives using `--prefix`. -When using --prefix, be careful to choose a good prefix - e.g. do not use a +you restrict its operation to a subset of the archives using ``--prefix``. +When using ``--prefix``, be careful to choose a good prefix - e.g. do not use a prefix "foo" if you do not also want to match "foobar". -It is strongly recommended to always run `prune --dry-run ...` first so you +It is strongly recommended to always run ``prune --dry-run ...`` first so you will see what it would do without it actually doing anything. :: @@ -457,7 +457,7 @@ Here are misc. notes about topics that are maybe not covered in enough detail in Item flags ~~~~~~~~~~ -`borg create -v --list` outputs a verbose list of all files, directories and other +``borg create -v --list`` outputs a verbose list of all files, directories and other file system items it considered (no matter whether they had content changes or not). For each item, it prefixes a single-letter flag that indicates type and/or status of the item. @@ -501,12 +501,12 @@ resource usage (RAM and disk space) as the amount of resources needed is (also) determined by the total amount of chunks in the repository (see `Indexes / Caches memory usage` for details). -`--chunker-params=10,23,16,4095 (default)` results in a fine-grained deduplication +``--chunker-params=10,23,16,4095 (default)`` results in a fine-grained deduplication and creates a big amount of chunks and thus uses a lot of resources to manage them. This is good for relatively small data volumes and if the machine has a good amount of free RAM and disk space. -`--chunker-params=19,23,21,4095` results in a coarse-grained deduplication and +``--chunker-params=19,23,21,4095`` results in a coarse-grained deduplication and creates a much smaller amount of chunks and thus uses less resources. This is good for relatively big data volumes and if the machine has a relatively low amount of free RAM and disk space. From 83c5321f53449dbea5f66b55d443d46cb1a2047b Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 20:13:45 +0100 Subject: [PATCH 224/321] Fixed RST warning markup. --- docs/usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 5d93be4b..0d39c89b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -29,7 +29,7 @@ Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL While you can set misc. log levels, do not expect that every command will give different output on different log levels - it's just a possibility. -..warning:: While some options (like ``--stats`` or ``--list``) will emit more +.. warning:: While some options (like ``--stats`` or ``--list``) will emit more informational messages, you have to use INFO (or lower) log level to make them show up in log output. Use ``-v`` or a logging configuration. From 1f1ff6137587b1ccf73fc917d349260f0eb1a67f Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 21:15:26 +0100 Subject: [PATCH 225/321] borg prune only knows "--keep-within" and not "--within". --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9cb7e3d1..a3cd2922 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -466,7 +466,7 @@ class Archiver: cache = Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0 and args.within is None: - self.print_error('At least one of the "within", "keep-hourly", "keep-daily", "keep-weekly", ' + self.print_error('At least one of the "keep-within", "keep-hourly", "keep-daily", "keep-weekly", ' '"keep-monthly" or "keep-yearly" settings must be specified') return self.exit_code if args.prefix: From 89ce86a33b80bf6c8c1062fad09b9a9261e1c681 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 22:23:54 +0100 Subject: [PATCH 226/321] =?UTF-8?q?Fixed=20more=20in=20usage.rst.=20-=20?= =?UTF-8?q?=E2=86=92=20=E2=80=93;=20=E2=80=93=20=E2=86=92=20--=20(CLI);=20?= =?UTF-8?q?Bullet=20list.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/usage.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 0d39c89b..3903f10b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -463,11 +463,11 @@ or not). For each item, it prefixes a single-letter flag that indicates type and/or status of the item. If you are interested only in a subset of that output, you can give e.g. -`--filter=AME` and it will only show regular files with A, M or E status (see +``--filter=AME`` and it will only show regular files with A, M or E status (see below). A uppercase character represents the status of a regular file relative to the -"files" cache (not relative to the repo - this is an issue if the files cache +"files" cache (not relative to the repo -- this is an issue if the files cache is not used). Metadata is stored in any case and for 'A' and 'M' also new data chunks are stored. For 'U' all data chunks refer to already existing chunks. @@ -519,10 +519,11 @@ In the worst case (all files are big and were touched in between backups), this will store all content into the repository again. Usually, it is not that bad though: + - usually most files are not touched, so it will just re-use the old chunks -it already has in the repo + it already has in the repo - files smaller than the (both old and new) minimum chunksize result in only -one chunk anyway, so the resulting chunks are same and deduplication will apply + one chunk anyway, so the resulting chunks are same and deduplication will apply If you switch chunker params to save resources for an existing repo that already has some backup archives, you will see an increasing effect over time, @@ -556,7 +557,7 @@ You need to be careful with what you give as filename when using ``--read-specia e.g. if you give ``/dev/zero``, your backup will never terminate. The given files' metadata is saved as it would be saved without -``--read-special`` (e.g. its name, its size [might be 0], its mode, etc.) - but +``--read-special`` (e.g. its name, its size [might be 0], its mode, etc.) -- but additionally, also the content read from it will be saved for it. Restoring such files' content is currently only supported one at a time via From 576348a9d4878c1f4c215a34e45eac2ef5667bf7 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 22:31:08 +0100 Subject: [PATCH 227/321] Use HTTPS everywhere. Especially when the website already redirects to HTTPS. --- README.rst | 2 +- borg/testsuite/archiver.py | 2 +- docs/deployment.rst | 2 +- docs/faq.rst | 2 +- docs/global.rst.inc | 10 +++++----- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 780a914e..d215f9ea 100644 --- a/README.rst +++ b/README.rst @@ -165,7 +165,7 @@ for the complete license. .. |doc| image:: https://readthedocs.org/projects/borgbackup/badge/?version=stable :alt: Documentation - :target: http://borgbackup.readthedocs.org/en/stable/ + :target: https://borgbackup.readthedocs.org/en/stable/ .. |build| image:: https://travis-ci.org/borgbackup/borg.svg :alt: Build Status diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d264224a..07279329 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -769,7 +769,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', '-v', '--list', self.repository_location + '::test1', 'input') self.assert_in("U input/file1", output) # this is expected, although surprising, for why, see: - # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file + # https://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file self.assert_in("A input/file2", output) def test_create_topical(self): diff --git a/docs/deployment.rst b/docs/deployment.rst index f60467df..06ada896 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -161,4 +161,4 @@ See also -------- * `SSH Daemon manpage `_ -* `Ansible `_ +* `Ansible `_ diff --git a/docs/faq.rst b/docs/faq.rst index bebb291f..94e00618 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -89,7 +89,7 @@ key file based encryption with a blank passphrase. See ``export`` in a shell script file should be safe, however, as the environment of a process is `accessible only to that user - `_. + `_. When backing up to remote encrypted repos, is encryption done locally? ---------------------------------------------------------------------- diff --git a/docs/global.rst.inc b/docs/global.rst.inc index 317c8d85..439b71d1 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -8,17 +8,17 @@ .. _issue tracker: https://github.com/borgbackup/borg/issues .. _deduplication: https://en.wikipedia.org/wiki/Data_deduplication .. _AES: https://en.wikipedia.org/wiki/Advanced_Encryption_Standard -.. _HMAC-SHA256: http://en.wikipedia.org/wiki/HMAC +.. _HMAC-SHA256: https://en.wikipedia.org/wiki/HMAC .. _SHA256: https://en.wikipedia.org/wiki/SHA-256 .. _PBKDF2: https://en.wikipedia.org/wiki/PBKDF2 .. _ACL: https://en.wikipedia.org/wiki/Access_control_list -.. _libacl: http://savannah.nongnu.org/projects/acl/ -.. _libattr: http://savannah.nongnu.org/projects/attr/ +.. _libacl: https://savannah.nongnu.org/projects/acl/ +.. _libattr: https://savannah.nongnu.org/projects/attr/ .. _liblz4: https://github.com/Cyan4973/lz4 .. _OpenSSL: https://www.openssl.org/ -.. _`Python 3`: http://www.python.org/ +.. _`Python 3`: https://www.python.org/ .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash -.. _msgpack: http://msgpack.org/ +.. _msgpack: https://msgpack.org/ .. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/ .. _llfuse: https://pypi.python.org/pypi/llfuse/ .. _homebrew: http://brew.sh/ From 8b9ae0ae92fcea21b751d2419952c8d2938f9c4f Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 17 Jan 2016 23:49:54 +0100 Subject: [PATCH 228/321] =?UTF-8?q?More=20=E2=80=93=20=E2=86=92=20--=20(CL?= =?UTF-8?q?I)=20fixes.=20Fixed=20spelling.=20AES-256=20is=20used.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * According to the comment in `borg/key.py`. --- docs/faq.rst | 6 +++--- docs/internals.rst | 19 +++++++++---------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 94e00618..acd6d859 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -67,14 +67,14 @@ adjust the level or algorithm). |project_name| offers a lot of different compression algorithms and levels. Which of them is the best for you pretty much depends on your -use case, your data, your hardware - so you need to do an informed +use case, your data, your hardware -- so you need to do an informed decision about whether you want to use compression, which algorithm and which level you want to use. This is why compression defaults to none. How can I specify the encryption passphrase programmatically? ------------------------------------------------------------- - + The encryption passphrase can be specified programmatically using the `BORG_PASSPHRASE` environment variable. This is convenient when setting up automated encrypted backups. Another option is to use @@ -93,7 +93,7 @@ key file based encryption with a blank passphrase. See When backing up to remote encrypted repos, is encryption done locally? ---------------------------------------------------------------------- - + Yes, file and directory metadata and data is locally encrypted, before leaving the local machine. We do not mean the transport layer encryption by that, but the data/metadata itself. Transport layer encryption (e.g. diff --git a/docs/internals.rst b/docs/internals.rst index 468e38b7..c8008f65 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -207,7 +207,7 @@ The |project_name| chunker uses a rolling hash computed by the Buzhash_ algorith It triggers (chunks) when the last HASH_MASK_BITS bits of the hash are zero, producing chunks of 2^HASH_MASK_BITS Bytes on average. -create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE +``borg create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` can be used to tune the chunker parameters, the default is: - CHUNK_MIN_EXP = 10 (minimum chunk size = 2^10 B = 1 kiB) @@ -311,28 +311,27 @@ more chunks than estimated above, because 1 file is at least 1 chunk). If a remote repository is used the repo index will be allocated on the remote side. -E.g. backing up a total count of 1Mi files with a total size of 1TiB. +E.g. backing up a total count of 1MiB files with a total size of 1TiB. -a) with create --chunker-params 10,23,16,4095 (default): +a) with create ``--chunker-params 10,23,16,4095`` (default): mem_usage = 2.8GiB -b) with create --chunker-params 10,23,20,4095 (custom): +b) with create ``--chunker-params 10,23,20,4095`` (custom): mem_usage = 0.4GiB -Note: there is also the --no-files-cache option to switch off the files cache. -You'll save some memory, but it will need to read / chunk all the files then as -it can not skip unmodified files then. - +.. note:: There is also the ``--no-files-cache`` option to switch off the files cache. + You'll save some memory, but it will need to read / chunk all the files as + it can not skip unmodified files then. Encryption ---------- -AES_ is used in CTR mode (so no need for padding). A 64bit initialization +AES_-256 is used in CTR mode (so no need for padding). A 64bit initialization vector is used, a `HMAC-SHA256`_ is computed on the encrypted chunk with a random 64bit nonce and both are stored in the chunk. -The header of each chunk is : ``TYPE(1)`` + ``HMAC(32)`` + ``NONCE(8)`` + ``CIPHERTEXT``. +The header of each chunk is: ``TYPE(1)`` + ``HMAC(32)`` + ``NONCE(8)`` + ``CIPHERTEXT``. Encryption and HMAC use two different keys. In AES CTR mode you can think of the IV as the start value for the counter. From 1f49d16a71e9efbb773b5940d194ec9a5d0c1201 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Mon, 18 Jan 2016 09:00:07 +0100 Subject: [PATCH 229/321] Fixed my changes. Thanks to @ThomasWaldmann for the review! --- docs/internals.rst | 2 +- docs/usage.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index c8008f65..5ea465a3 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -311,7 +311,7 @@ more chunks than estimated above, because 1 file is at least 1 chunk). If a remote repository is used the repo index will be allocated on the remote side. -E.g. backing up a total count of 1MiB files with a total size of 1TiB. +E.g. backing up a total count of 1 million files with a total size of 1TiB. a) with create ``--chunker-params 10,23,16,4095`` (default): diff --git a/docs/usage.rst b/docs/usage.rst index 3903f10b..73a60dc8 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -75,7 +75,7 @@ Some "yes" sayers (if set, they automatically confirm that you really want to do BORG_RELOCATED_REPO_ACCESS_IS_OK For "Warning: The repository at location ... was previously located at ..." BORG_CHECK_I_KNOW_WHAT_I_AM_DOING - For "Warning: 'check ``--repair``' is an experimental feature that might result in data loss." + For "Warning: '``check --repair``' is an experimental feature that might result in data loss." BORG_DELETE_I_KNOW_WHAT_I_AM_DOING For "You requested to completely DELETE the repository *including* all archives it contains: " From 665c3db2e9128a09708f60e757d527ad9bb280ec Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Mon, 18 Jan 2016 09:32:51 +0100 Subject: [PATCH 230/321] Improved understandability of sentence in deployment.rst. Thanks to @xor-gate. Related to https://github.com/borgbackup/borg/pull/529/files#r49952612 --- docs/deployment.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index 06ada896..bd69f943 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -73,8 +73,10 @@ The options which are added to the key will perform the following: 3. Run ``borg serve`` restricted at the client base path 4. Restrict ssh and do not allow stuff which imposes a security risk -Due to the cd command we use, the server automatically changes the current working -directory so the client will not need to append the hostname to the remote URI. +Due to the ``cd`` command we use, the server automatically changes the current +working directory. Then client doesn't need to have knowledge of the absolute +or relative remote repository path and can directly access the repositories at +``@:``. .. note:: The setup above ignores all client given commandline parameters which are normally appended to the `borg serve` command. From 32900c867945be19d692ad72dde627e5aeffd186 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Mon, 18 Jan 2016 10:38:55 +0100 Subject: [PATCH 231/321] `Mi` does stand for a IEC binary prefix e.g. 2^20. --- docs/internals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/internals.rst b/docs/internals.rst index 5ea465a3..246ffca9 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -311,7 +311,7 @@ more chunks than estimated above, because 1 file is at least 1 chunk). If a remote repository is used the repo index will be allocated on the remote side. -E.g. backing up a total count of 1 million files with a total size of 1TiB. +E.g. backing up a total count of 1 Mi (IEC binary prefix e.g. 2^20) files with a total size of 1TiB. a) with create ``--chunker-params 10,23,16,4095`` (default): From 4c00bb0d2f080fd33dad531d58f65dd11377bb0b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 18 Jan 2016 14:30:41 +0100 Subject: [PATCH 232/321] fix crash when using borg create --dry-run --keep-tag-files, fixes #570 --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9cb7e3d1..2f94aa14 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -228,7 +228,7 @@ class Archiver: elif stat.S_ISDIR(st.st_mode): tag_paths = dir_is_tagged(path, exclude_caches, exclude_if_present) if tag_paths: - if keep_tag_files: + if keep_tag_files and not dry_run: archive.process_dir(path, st) for tag_path in tag_paths: self._process(archive, cache, excludes, exclude_caches, exclude_if_present, From 7851df089a62515b09b45c15d584be2ab9ea0dec Mon Sep 17 00:00:00 2001 From: Piotr Pawlow Date: Mon, 18 Jan 2016 14:35:11 +0100 Subject: [PATCH 233/321] Disable unneeded SSH features in authorized_keys example for security. --- docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 18457e9c..ca7acc79 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -207,7 +207,7 @@ the remote server's authorized_keys file. Only the forced command will be run when the key authenticates a connection. This example will start |project_name| in server mode, and limit the |project_name| server to a specific filesystem path:: - command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...] + command="borg serve --restrict-to-path /mnt/backup",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-rsa AAAAB3[...] If it is not possible to install |project_name| on the remote host, it is still possible to use the remote host to store a repository by From 897c763a5b703eaf4c7addb645d856cf2a6b06a1 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Mon, 18 Jan 2016 16:30:27 +0100 Subject: [PATCH 234/321] Fix markup in README.rst Github generated wrong markup, because of a few whitespace errors. This patch fixes the rst markup of the README.rst file. --- README.rst | 63 +++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/README.rst b/README.rst index d215f9ea..4ac44031 100644 --- a/README.rst +++ b/README.rst @@ -30,20 +30,15 @@ Main features Compared to other deduplication approaches, this method does NOT depend on: - * file/directory names staying the same + * file/directory names staying the same: So you can move your stuff around + without killing the deduplication, even between machines sharing a repo. - So you can move your stuff around without killing the deduplication, - even between machines sharing a repo. + * complete files or time stamps staying the same: If a big file changes a + little, only a few new chunks need to be stored - this is great for VMs or + raw disks. - * complete files or time stamps staying the same - - If a big file changes a little, only a few new chunks will be stored - - this is great for VMs or raw disks. - - * the absolute position of a data chunk inside a file - - Stuff may get shifted and will still be found by the deduplication - algorithm. + * The absolute position of a data chunk inside a file: Stuff may get shifted + and will still be found by the deduplication algorithm. **Speed** * performance critical code (chunking, compression, encryption) is @@ -109,15 +104,15 @@ For a graphical frontend refer to our complementary project `BorgWeb `_ - * `Releases `_ - * `PyPI packages `_ - * `ChangeLog `_ - * `GitHub `_ - * `Issue Tracker `_ - * `Bounties & Fundraisers `_ - * `Mailing List `_ - * `License `_ +* `Main Web Site `_ +* `Releases `_ +* `PyPI packages `_ +* `ChangeLog `_ +* `GitHub `_ +* `Issue Tracker `_ +* `Bounties & Fundraisers `_ +* `Mailing List `_ +* `License `_ Notes ----- @@ -132,19 +127,19 @@ Differences between Attic and Borg Here's a (incomplete) list of some major changes: - * more open, faster paced development (see `issue #1 `_) - * lots of attic issues fixed (see `issue #5 `_) - * less chunk management overhead via --chunker-params option (less memory and disk usage) - * faster remote cache resync (useful when backing up multiple machines into same repo) - * compression: no, lz4, zlib or lzma compression, adjustable compression levels - * repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode) - * simple sparse file support, great for virtual machine disk files - * can read special files (e.g. block devices) or from stdin, write to stdout - * mkdir-based locking is more compatible than attic's posix locking - * uses fadvise to not spoil / blow up the fs cache - * better error messages / exception handling - * better logging, screen output, progress indication - * tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X +* more open, faster paced development (see `issue #1 `_) +* lots of attic issues fixed (see `issue #5 `_) +* less chunk management overhead via --chunker-params option (less memory and disk usage) +* faster remote cache resync (useful when backing up multiple machines into same repo) +* compression: no, lz4, zlib or lzma compression, adjustable compression levels +* repokey replaces problematic passphrase mode (you can't change the passphrase nor the pbkdf2 iteration count in "passphrase" mode) +* simple sparse file support, great for virtual machine disk files +* can read special files (e.g. block devices) or from stdin, write to stdout +* mkdir-based locking is more compatible than attic's posix locking +* uses fadvise to not spoil / blow up the fs cache +* better error messages / exception handling +* better logging, screen output, progress indication +* tested on misc. Linux systems, 32 and 64bit, FreeBSD, OpenBSD, NetBSD, Mac OS X Please read the `ChangeLog`_ (or ``CHANGES.rst`` in the source distribution) for more information. From efec1a396ee80a818527e89ddf381f5ae2d3bdcf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 18 Jan 2016 18:25:10 +0100 Subject: [PATCH 235/321] Vagrantfile: use pyinstaller 3.1, freebsd sqlite3 fix, fixes #569 --- Vagrantfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 173a3644..165fe4fa 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -76,7 +76,7 @@ def packages_freebsd pkg install -y openssl liblz4 fusefs-libs pkgconf pkg install -y fakeroot git bash # for building python: - pkg install sqlite3 + pkg install -y sqlite3 # make bash default / work: chsh -s bash vagrant mount -t fdescfs fdesc /dev/fd @@ -211,8 +211,7 @@ def install_pyinstaller(boxname) . borg-env/bin/activate git clone https://github.com/pyinstaller/pyinstaller.git cd pyinstaller - # use develop branch for now, see borgbackup issue #336 - git checkout develop + git checkout v3.1 pip install -e . EOF end @@ -224,8 +223,7 @@ def install_pyinstaller_bootloader(boxname) . borg-env/bin/activate git clone https://github.com/pyinstaller/pyinstaller.git cd pyinstaller - # use develop branch for now, see borgbackup issue #336 - git checkout develop + git checkout v3.1 # build bootloader, if it is not included cd bootloader python ./waf all From a3fa965ded378706a20babf9a328f6ccc3acad6b Mon Sep 17 00:00:00 2001 From: Piotr Pawlow Date: Mon, 18 Jan 2016 18:39:11 +0100 Subject: [PATCH 236/321] Added no-agent-forwarding,no-user-rc to SSH key options. --- docs/deployment.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index bd69f943..cffb0831 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -62,7 +62,8 @@ forced command and restrictions applied as shown below: command="cd /home/backup/repos/; borg serve --restrict-path /home/backup/repos/", - no-port-forwarding,no-X11-forwarding,no-pty + no-port-forwarding,no-X11-forwarding,no-pty, + no-agent-forwarding,no-user-rc .. note:: The text shown above needs to be written on a single line! @@ -141,7 +142,7 @@ package manager to install and keep borg up-to-date. - file: path="{{ pool }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory - authorized_key: user="{{ user }}" key="{{ item.key }}" - key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty' + key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc' with_items: auth_users - file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file - file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory From 987aaa34dfed5067c5cb1fb679f3a6de50ccf0af Mon Sep 17 00:00:00 2001 From: Piotr Pawlow Date: Mon, 18 Jan 2016 18:49:07 +0100 Subject: [PATCH 237/321] Added SSH key options to the usage example. --- docs/usage.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 73a60dc8..e3a9ed7b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -419,9 +419,10 @@ Examples :: # Allow an SSH keypair to only run borg, and only have access to /mnt/backup. + # Use key options to disable unneeded and potentially dangerous SSH functionality. # This will help to secure an automated remote backup system. $ cat ~/.ssh/authorized_keys - command="borg serve --restrict-to-path /mnt/backup" ssh-rsa AAAAB3[...] + command="borg serve --restrict-to-path /mnt/backup",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-rsa AAAAB3[...] .. include:: usage/upgrade.rst.inc From 560a61b63401c8644686c54f94566d531d12fa68 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 15:27:37 +0100 Subject: [PATCH 238/321] Add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 31910fb6..077386e2 100644 --- a/AUTHORS +++ b/AUTHORS @@ -5,6 +5,7 @@ Borg Contributors ("The Borg Collective") - Antoine Beaupré - Radek Podgorny - Yuri D'Elia +- Michael Hanselmann Borg is a fork of Attic. From c1feb4b532bbc87321d0264899d699ecb0962891 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Fri, 15 Jan 2016 22:11:47 +0100 Subject: [PATCH 239/321] Simplify pattern tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Stop using “adjust_pattern” and “exclude_path” as they're utility functions not relevant to testing pattern classes - Cover a few more cases, especially with more than one path separator and relative paths - At least one dedicated test function for each pattern style as opposed to a single, big test mixing styles - Use positive instead of negative matching (i.e. the expected list of resulting items is a list of items matching a pattern) --- borg/testsuite/helpers.py | 127 ++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 46 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 3da955d7..725dba9b 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -9,7 +9,7 @@ import sys import msgpack import msgpack.fallback -from ..helpers import adjust_patterns, exclude_path, Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ +from ..helpers import exclude_path, Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern @@ -160,70 +160,105 @@ class FormatTimedeltaTestCase(BaseTestCase): ) -def check_patterns(files, paths, excludes, expected): - """Utility for testing exclusion patterns. +def check_patterns(files, pattern, expected): + """Utility for testing patterns. """ - patterns = adjust_patterns(paths, excludes) - included = [path for path in files if not exclude_path(path, patterns)] + assert all([f == os.path.normpath(f) for f in files]), \ + "Pattern matchers expect normalized input paths" - assert included == (files if expected is None else expected) + matched = [f for f in files if pattern.match(f)] + + assert matched == (files if expected is None else expected) -@pytest.mark.parametrize("paths, excludes, expected", [ - # "None" means all files, i.e. none excluded - ([], [], None), - (['/'], [], None), - (['/'], ['/h'], None), - (['/'], ['/home'], ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']), - (['/'], ['/home/'], ['/etc/passwd', '/etc/hosts', '/home', '/var/log/messages', '/var/log/dmesg']), - (['/home/u'], [], []), - (['/', '/home', '/etc/hosts'], ['/'], []), - (['/home/'], ['/home/user2'], ['/home', '/home/user/.profile', '/home/user/.bashrc']), - (['/'], ['*.profile', '/var/log'], - ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc', '/home/user2/public_html/index.html']), - (['/'], ['/home/*/public_html', '*.profile', '*/log/*'], - ['/etc/passwd', '/etc/hosts', '/home', '/home/user/.bashrc']), - (['/etc/', '/var'], ['dmesg'], ['/etc/passwd', '/etc/hosts', '/var/log/messages', '/var/log/dmesg']), +@pytest.mark.parametrize("pattern, expected", [ + # "None" means all files, i.e. all match the given pattern + ("/", None), + ("/./", None), + ("", []), + ("/home/u", []), + ("/home/user", ["/home/user/.profile", "/home/user/.bashrc"]), + ("/etc", ["/etc/server/config", "/etc/server/hosts"]), + ("///etc//////", ["/etc/server/config", "/etc/server/hosts"]), + ("/./home//..//home/user2", ["/home/user2/.profile", "/home/user2/public_html/index.html"]), + ("/srv", ["/srv/messages", "/srv/dmesg"]), ]) -def test_patterns(paths, excludes, expected): +def test_patterns_prefix(pattern, expected): files = [ - '/etc/passwd', '/etc/hosts', '/home', - '/home/user/.profile', '/home/user/.bashrc', - '/home/user2/.profile', '/home/user2/public_html/index.html', - '/var/log/messages', '/var/log/dmesg', + "/etc/server/config", "/etc/server/hosts", "/home", "/home/user/.profile", "/home/user/.bashrc", + "/home/user2/.profile", "/home/user2/public_html/index.html", "/srv/messages", "/srv/dmesg", ] - check_patterns(files, paths, [FnmatchPattern(p) for p in excludes], expected) + check_patterns(files, PathPrefixPattern(pattern), expected) -@pytest.mark.parametrize("paths, excludes, expected", [ - # "None" means all files, i.e. none excluded - ([], [], None), - (['/'], [], None), - (['/'], ['.*'], []), - (['/'], ['^/'], []), - (['/'], ['^abc$'], None), - (['/'], ['^(?!/home/)'], - ['/home/user/.profile', '/home/user/.bashrc', '/home/user2/.profile', - '/home/user2/public_html/index.html']), +@pytest.mark.parametrize("pattern, expected", [ + # "None" means all files, i.e. all match the given pattern + ("", []), + ("foo", []), + ("relative", ["relative/path1", "relative/two"]), + ("more", ["more/relative"]), ]) -def test_patterns_regex(paths, excludes, expected): +def test_patterns_prefix_relative(pattern, expected): + files = ["relative/path1", "relative/two", "more/relative"] + + check_patterns(files, PathPrefixPattern(pattern), expected) + + +@pytest.mark.parametrize("pattern, expected", [ + # "None" means all files, i.e. all match the given pattern + ("/*", None), + ("/./*", None), + ("*", None), + ("*/*", None), + ("*///*", None), + ("/home/u", []), + ("/home/*", + ["/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile", "/home/user2/public_html/index.html", + "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails"]), + ("/home/user/*", ["/home/user/.profile", "/home/user/.bashrc"]), + ("/etc/*", ["/etc/server/config", "/etc/server/hosts"]), + ("*/.pr????e", ["/home/user/.profile", "/home/user2/.profile"]), + ("///etc//////*", ["/etc/server/config", "/etc/server/hosts"]), + ("/./home//..//home/user2/*", ["/home/user2/.profile", "/home/user2/public_html/index.html"]), + ("/srv*", ["/srv/messages", "/srv/dmesg"]), + ("/home/*/.thumbnails", ["/home/foo/.thumbnails", "/home/foo/bar/.thumbnails"]), + ]) +def test_patterns_fnmatch(pattern, expected): + files = [ + "/etc/server/config", "/etc/server/hosts", "/home", "/home/user/.profile", "/home/user/.bashrc", + "/home/user2/.profile", "/home/user2/public_html/index.html", "/srv/messages", "/srv/dmesg", + "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", + ] + + check_patterns(files, FnmatchPattern(pattern), expected) + + +@pytest.mark.parametrize("pattern, expected", [ + # "None" means all files, i.e. all match the given pattern + ("", None), + (".*", None), + ("^/", None), + ("^abc$", []), + ("^[^/]", []), + ("^(?!/srv|/foo|/opt)", + ["/home", "/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile", + "/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails",]), + ]) +def test_patterns_regex(pattern, expected): files = [ '/srv/data', '/foo/bar', '/home', '/home/user/.profile', '/home/user/.bashrc', '/home/user2/.profile', '/home/user2/public_html/index.html', '/opt/log/messages.txt', '/opt/log/dmesg.txt', + "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", ] - patterns = [] + obj = RegexPattern(pattern) + assert str(obj) == pattern + assert obj.pattern == pattern - for i in excludes: - pat = RegexPattern(i) - assert str(pat) == i - assert pat.pattern == i - patterns.append(pat) - - check_patterns(files, paths, patterns, expected) + check_patterns(files, obj, expected) def test_regex_pattern(): From 9747755131fadf909069d7a84dc793267af2c5ff Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 13:32:49 +0100 Subject: [PATCH 240/321] Add pattern matcher wrapper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The utility functions “adjust_patterns” and “exclude_path” produce respectively use a standard list object containing pattern objects. With the forthcoming introduction of patterns for filtering files to be extracted it's better to move the logic of these classes into a single class. The wrapper allows adding any number of patterns to an internal list together with a value to be returned if a match function finds that one of the patterns matches. A fallback value is returned otherwise. --- borg/helpers.py | 21 +++++++++++++++++++++ borg/testsuite/helpers.py | 25 ++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index ce344e3f..39aad553 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -274,6 +274,27 @@ def exclude_path(path, patterns): return False +class PatternMatcher: + def __init__(self, fallback=None): + self._items = [] + + # Value to return from match function when none of the patterns match. + self.fallback = fallback + + def add(self, patterns, value): + """Add list of patterns to internal list. The given value is returned from the match function when one of the + given patterns matches. + """ + self._items.extend((i, value) for i in patterns) + + def match(self, path): + for (pattern, value) in self._items: + if pattern.match(path): + return value + + return self.fallback + + def normalized(func): """ Decorator for the Pattern match methods, returning a wrapper that normalizes OSX paths to match the normalized pattern on OSX, and diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 725dba9b..ee8a9e91 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -12,7 +12,7 @@ import msgpack.fallback from ..helpers import exclude_path, Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern + ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher from . import BaseTestCase, environment_variable, FakeInputs @@ -374,6 +374,29 @@ def test_parse_pattern_error(pattern): parse_pattern(pattern) +def test_pattern_matcher(): + pm = PatternMatcher() + + assert pm.fallback is None + + for i in ["", "foo", "bar"]: + assert pm.match(i) is None + + pm.add([RegexPattern("^a")], "A") + pm.add([RegexPattern("^b"), RegexPattern("^z")], "B") + pm.add([RegexPattern("^$")], "Empty") + pm.fallback = "FileNotFound" + + assert pm.match("") == "Empty" + assert pm.match("aaa") == "A" + assert pm.match("bbb") == "B" + assert pm.match("ccc") == "FileNotFound" + assert pm.match("xyz") == "FileNotFound" + assert pm.match("z") == "B" + + assert PatternMatcher(fallback="hey!").fallback == "hey!" + + def test_compression_specs(): with pytest.raises(ValueError): CompressionSpec('') From 190107ada77156e3703d3b7589b911c2d6da6dff Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 15:11:28 +0100 Subject: [PATCH 241/321] =?UTF-8?q?Replace=20use=20of=20=E2=80=9Cexclude?= =?UTF-8?q?=5Fpath=E2=80=9D=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The newly added pattern matcher class can replace the “exclude_path” function. The latter is going to be removed in a later change. --- borg/testsuite/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index ee8a9e91..22bfe903 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -9,7 +9,7 @@ import sys import msgpack import msgpack.fallback -from ..helpers import exclude_path, Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ +from ..helpers import Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher @@ -334,8 +334,9 @@ def test_patterns_from_file(tmpdir, lines, expected): ] def evaluate(filename): - patterns = load_excludes(open(filename, "rt")) - return [path for path in files if not exclude_path(path, patterns)] + matcher = PatternMatcher(fallback=True) + matcher.add(load_excludes(open(filename, "rt")), False) + return [path for path in files if matcher.match(path)] exclfile = tmpdir.join("exclude.txt") From 1fa4d2e516f8334f7bcf6f9bdc78fe6aafd45e65 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 13:05:48 +0100 Subject: [PATCH 242/321] Use constants for pattern style prefixes The prefix used for pattern styles should be kept together with the respective style implementation. --- borg/helpers.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 39aad553..eb3231de 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -316,6 +316,8 @@ def normalized(func): class PatternBase: """Shared logic for inclusion/exclusion patterns. """ + PREFIX = NotImplemented + def __init__(self, pattern): self.pattern_orig = pattern self.match_count = 0 @@ -360,6 +362,8 @@ class PathPrefixPattern(PatternBase): If a directory is specified, all paths that start with that path match as well. A trailing slash makes no difference. """ + PREFIX = "pp" + def _prepare(self, pattern): self.pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep @@ -371,6 +375,8 @@ class FnmatchPattern(PatternBase): """Shell glob patterns to exclude. A trailing slash means to exclude the contents of a directory, but not the directory itself. """ + PREFIX = "fm" + def _prepare(self, pattern): if pattern.endswith(os.path.sep): pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep @@ -390,6 +396,8 @@ class FnmatchPattern(PatternBase): class RegexPattern(PatternBase): """Regular expression to exclude. """ + PREFIX = "re" + def _prepare(self, pattern): self.pattern = pattern self.regex = re.compile(pattern) @@ -402,11 +410,12 @@ class RegexPattern(PatternBase): return (self.regex.search(path) is not None) -_DEFAULT_PATTERN_STYLE = "fm" -_PATTERN_STYLES = { - "fm": FnmatchPattern, - "re": RegexPattern, - } +_PATTERN_STYLES = set([ + FnmatchPattern, + RegexPattern, +]) + +_PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES) def parse_pattern(pattern): @@ -415,9 +424,9 @@ def parse_pattern(pattern): if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum(): (style, pattern) = (pattern[:2], pattern[3:]) else: - style = _DEFAULT_PATTERN_STYLE + style = FnmatchPattern.PREFIX - cls = _PATTERN_STYLES.get(style, None) + cls = _PATTERN_STYLE_BY_PREFIX.get(style, None) if cls is None: raise ValueError("Unknown pattern style: {}".format(style)) From b6362b596390651d244086907702185745c4bc55 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 13:08:49 +0100 Subject: [PATCH 243/321] Flexible default pattern style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A function to parse pattern specifications was introduced in commit 2bafece. Since then it had a hardcoded default style of “fm”, meaning fnmatch. With the forthcoming support for extracting files using patterns this default style must be more flexible. --- borg/helpers.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index eb3231de..adb75fb4 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -418,18 +418,18 @@ _PATTERN_STYLES = set([ _PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES) -def parse_pattern(pattern): +def parse_pattern(pattern, fallback=FnmatchPattern): """Read pattern from string and return an instance of the appropriate implementation class. """ if len(pattern) > 2 and pattern[2] == ":" and pattern[:2].isalnum(): (style, pattern) = (pattern[:2], pattern[3:]) + + cls = _PATTERN_STYLE_BY_PREFIX.get(style, None) + + if cls is None: + raise ValueError("Unknown pattern style: {}".format(style)) else: - style = FnmatchPattern.PREFIX - - cls = _PATTERN_STYLE_BY_PREFIX.get(style, None) - - if cls is None: - raise ValueError("Unknown pattern style: {}".format(style)) + cls = fallback return cls(pattern) From 848375e2fe35ecddfcdfd7bb30811fff87b549da Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 13:09:08 +0100 Subject: [PATCH 244/321] Add and document path prefix as pattern style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The “extract” command supports extracting all files underneath a given set of prefix paths. The forthcoming support for extracting files using a pattern (i.e. only files ending in “.zip”) requires the introduction of path prefixes as a third pattern style, making it also available for exclusions. --- borg/archiver.py | 16 ++++++++++------ borg/helpers.py | 1 + borg/testsuite/helpers.py | 10 ++++++++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 57d311bc..66298fe0 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -611,12 +611,12 @@ class Archiver: helptext = {} helptext['patterns'] = textwrap.dedent(''' - Exclusion patterns support two separate styles, fnmatch and regular - expressions. If followed by a colon (':') the first two characters of - a pattern are used as a style selector. Explicit style selection is necessary - when regular expressions are desired or when the desired fnmatch pattern - starts with two alphanumeric characters followed by a colon (i.e. - `aa:something/*`). + Exclusion patterns support three separate styles, fnmatch, regular + expressions and path prefixes. If followed by a colon (':') the first two + characters of a pattern are used as a style selector. Explicit style + selection is necessary when a non-default style is desired or when the + desired pattern starts with two alphanumeric characters followed by a colon + (i.e. `aa:something/*`). `Fnmatch `_ patterns use a variant of shell pattern syntax, with '*' matching any number of @@ -640,6 +640,10 @@ class Archiver: documentation for the re module `_. + Prefix path patterns can be selected with the prefix `pp:`. This pattern + style is useful to match whole sub-directories. The pattern `pp:/data/bar` + matches `/data/bar` and everything therein. + Exclusions can be passed via the command line option `--exclude`. When used from within a shell the patterns should be quoted to protect them from expansion. diff --git a/borg/helpers.py b/borg/helpers.py index adb75fb4..8c1bb594 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -412,6 +412,7 @@ class RegexPattern(PatternBase): _PATTERN_STYLES = set([ FnmatchPattern, + PathPrefixPattern, RegexPattern, ]) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 22bfe903..f31bd984 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -324,6 +324,10 @@ def test_invalid_unicode_pattern(pattern): ["/more/data"]), ([r" re:^\s "], ["/data/something00.txt", "/more/data", "/home", "/whitespace/end\t"]), ([r" re:\s$ "], ["/data/something00.txt", "/more/data", "/home", " #/wsfoobar", "\tstart/whitespace"]), + (["pp:./"], None), + (["pp:/"], [" #/wsfoobar", "\tstart/whitespace"]), + (["pp:aaabbb"], None), + (["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["/more/data", "/home"]), ]) def test_patterns_from_file(tmpdir, lines, expected): files = [ @@ -364,6 +368,12 @@ def test_patterns_from_file(tmpdir, lines, expected): ("re:.*", RegexPattern), ("re:^/something/", RegexPattern), ("re:re:^/something/", RegexPattern), + + # Path prefix + ("pp:", PathPrefixPattern), + ("pp:/", PathPrefixPattern), + ("pp:/data/", PathPrefixPattern), + ("pp:pp:/data/", PathPrefixPattern), ]) def test_parse_pattern(pattern, cls): assert isinstance(parse_pattern(pattern), cls) From ceae4a9fa8fa030984c750fc8fcc795167ece639 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 16:45:42 +0100 Subject: [PATCH 245/321] Support patterns on extraction, fixes #361 This change implements the functionality requested in issue #361: extracting files with a given extension. It does so by permitting patterns to be used instead plain prefix paths. The pattern styles supported are the same as for exclusions. --- borg/archiver.py | 22 +++++++++++++++++----- borg/testsuite/archiver.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 66298fe0..d5c1d196 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -286,13 +286,25 @@ class Archiver: manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.location.archive, numeric_owner=args.numeric_owner) - patterns = adjust_patterns(args.paths, args.excludes) + + matcher = PatternMatcher() + if args.excludes: + matcher.add(args.excludes, False) + + include_patterns = [] + + if args.paths: + include_patterns.extend(parse_pattern(i, PathPrefixPattern) for i in args.paths) + matcher.add(include_patterns, True) + + matcher.fallback = not include_patterns + dry_run = args.dry_run stdout = args.stdout sparse = args.sparse strip_components = args.strip_components dirs = [] - for item in archive.iter_items(lambda item: not exclude_path(item[b'path'], patterns), preload=True): + for item in archive.iter_items(lambda item: matcher.match(item[b'path']), preload=True): orig_path = item[b'path'] if strip_components: item[b'path'] = os.sep.join(orig_path.split(os.sep)[strip_components:]) @@ -317,8 +329,8 @@ class Archiver: if not args.dry_run: while dirs: archive.extract_item(dirs.pop(-1)) - for pattern in (patterns or []): - if isinstance(pattern, PathPrefixPattern) and pattern.match_count == 0: + for pattern in include_patterns: + if pattern.match_count == 0: self.print_warning("Include pattern '%s' never matched.", pattern) return self.exit_code @@ -965,7 +977,7 @@ class Archiver: type=location_validator(archive=True), help='archive to extract') subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, - help='paths to extract') + help='paths to extract; patterns are supported') rename_epilog = textwrap.dedent(""" This command renames an archive in the repository. diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 07279329..f75cc120 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -562,6 +562,39 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', '--exclude-from=' + self.exclude_file_path, self.repository_location + '::test') self.assert_equal(sorted(os.listdir('output/input')), ['file3']) + def test_extract_with_pattern(self): + self.cmd("init", self.repository_location) + self.create_regular_file("file1", size=1024 * 80) + self.create_regular_file("file2", size=1024 * 80) + self.create_regular_file("file3", size=1024 * 80) + self.create_regular_file("file4", size=1024 * 80) + self.create_regular_file("file333", size=1024 * 80) + + self.cmd("create", self.repository_location + "::test", "input") + + # Extract everything with regular expression + with changedir("output"): + self.cmd("extract", self.repository_location + "::test", "re:.*") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333", "file4"]) + shutil.rmtree("output/input") + + # Extract with pattern while also excluding files + with changedir("output"): + self.cmd("extract", "--exclude=re:file[34]$", self.repository_location + "::test", r"re:file\d$") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"]) + shutil.rmtree("output/input") + + # Combine --exclude with pattern for extraction + with changedir("output"): + self.cmd("extract", "--exclude=input/file1", self.repository_location + "::test", "re:file[12]$") + self.assert_equal(sorted(os.listdir("output/input")), ["file2"]) + shutil.rmtree("output/input") + + # Multiple pattern + with changedir("output"): + self.cmd("extract", self.repository_location + "::test", "fm:input/file1", "fm:*file33*", "input/file2") + self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file333"]) + def test_exclude_caches(self): self.cmd('init', self.repository_location) self.create_regular_file('file1', size=1024 * 80) From dad0ba96619f76cd3f73499cf20d8e56c9212279 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Mon, 18 Jan 2016 16:45:48 +0100 Subject: [PATCH 246/321] Remove old-style pattern handling functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the “adjust_pattern” and “exclude_path” functions and replace them with the recently introduced pattern matcher class. --- borg/archiver.py | 19 ++++++++++++------- borg/helpers.py | 17 ----------------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index d5c1d196..62f00f50 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,11 +17,11 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, parse_pattern, PathPrefixPattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ + format_file_mode, parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ - EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi + EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher from .logger import create_logger, setup_logging logger = create_logger() from .compress import Compressor, COMPR_BUFFER @@ -129,6 +129,10 @@ class Archiver: def do_create(self, args): """Create new archive""" + matcher = PatternMatcher(fallback=True) + if args.excludes: + matcher.add(args.excludes, False) + def create_inner(archive, cache): # Add cache dir to inode_skip list skip_inodes = set() @@ -166,7 +170,7 @@ class Archiver: continue else: restrict_dev = None - self._process(archive, cache, args.excludes, args.exclude_caches, args.exclude_if_present, + self._process(archive, cache, matcher, args.exclude_caches, args.exclude_if_present, args.keep_tag_files, skip_inodes, path, restrict_dev, read_special=args.read_special, dry_run=dry_run) if not dry_run: @@ -202,11 +206,12 @@ class Archiver: create_inner(None, None) return self.exit_code - def _process(self, archive, cache, excludes, exclude_caches, exclude_if_present, + def _process(self, archive, cache, matcher, exclude_caches, exclude_if_present, keep_tag_files, skip_inodes, path, restrict_dev, read_special=False, dry_run=False): - if exclude_path(path, excludes): + if not matcher.match(path): return + try: st = os.lstat(path) except OSError as e: @@ -235,7 +240,7 @@ class Archiver: if keep_tag_files and not dry_run: archive.process_dir(path, st) for tag_path in tag_paths: - self._process(archive, cache, excludes, exclude_caches, exclude_if_present, + self._process(archive, cache, matcher, exclude_caches, exclude_if_present, keep_tag_files, skip_inodes, tag_path, restrict_dev, read_special=read_special, dry_run=dry_run) return @@ -249,7 +254,7 @@ class Archiver: else: for filename in sorted(entries): entry_path = os.path.normpath(os.path.join(path, filename)) - self._process(archive, cache, excludes, exclude_caches, exclude_if_present, + self._process(archive, cache, matcher, exclude_caches, exclude_if_present, keep_tag_files, skip_inodes, entry_path, restrict_dev, read_special=read_special, dry_run=dry_run) elif stat.S_ISLNK(st.st_mode): diff --git a/borg/helpers.py b/borg/helpers.py index 8c1bb594..91c9e043 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -257,23 +257,6 @@ def update_excludes(args): file.close() -def adjust_patterns(paths, excludes): - if paths: - return (excludes or []) + [PathPrefixPattern(path) for path in paths] + [FnmatchPattern('*')] - else: - return excludes - - -def exclude_path(path, patterns): - """Used by create and extract sub-commands to determine - whether or not an item should be processed. - """ - for pattern in (patterns or []): - if pattern.match(path): - return isinstance(pattern, (FnmatchPattern, RegexPattern)) - return False - - class PatternMatcher: def __init__(self, fallback=None): self._items = [] From 2dde49f0d4ba2e7670a4b9b84312c6f1324df2f1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 19 Jan 2016 01:02:14 +0100 Subject: [PATCH 247/321] update CHANGES --- docs/changes.rst | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index fb958aa9..4054bca4 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,69 @@ Changelog ========= +Version 0.30.0 (not released yet) +--------------------------------- + +Compatibility notes: + +- you may need to use -v (or --info) more often to actually see output emitted + at INFO log level (because it is suppressed at the default WARNING log level). + See the "general" section in the usage docs. +- see below about BORG_DELETE_I_KNOW_WHAT_I_AM_DOING (was: + BORG_CHECK_I_KNOW_WHAT_I_AM_DOING) + +Bug fixes: + +- fix crash when using borg create --dry-run --keep-tag-files, #570 +- make sure teardown with cleanup happens for Cache and RepositoryCache, + avoiding leftover locks and TEMP dir contents, #285 (partially), #548 +- fix locking KeyError, partial fix for #502 +- log stats consistently, #526 +- add abbreviated weekday to timestamp format, fixes #496 +- strip whitespace when loading exclusions from file +- unset LD_LIBRARY_PATH before invoking ssh, fixes strange OpenSSL library + version warning when using the borg binary, #514 +- add some error handling/fallback for C library loading, #494 +- added BORG_DELETE_I_KNOW_WHAT_I_AM_DOING for check in "borg delete", #503 +- remove unused "repair" rpc method name + +New features: + +- implement exclusions using regular expression patterns. +- support patterns for borg extract, #361 +- support different styles for patterns: + + - fnmatch (fm: prefix, default when omitted), like borg <= 0.29. + - regular expression (re:), new! +- --progress option for borg upgrade (#291) and borg delete +- update progress indication more often (e.g. within big files), #500 +- finer chunker granularity for items metadata stream, #547, #487 +- borg create --list now used (additionally to -v) to enable the verbose + file list output +- display borg version below tracebacks, #532 + +Other changes: + +- hashtable size (and thus: RAM and disk consumption) follows a growth policy: + grows fast while small, grows slower when getting bigger, #527 +- Vagrantfile: use pyinstaller 3.1 to build binaries, freebsd sqlite3 fix, + fixes #569 +- docs: + + - important: clarify -v and log levels in usage -> general, please read! + - sphinx configuration: create a simple man page from usage docs + - add a repo server setup example + - disable unneeded SSH features in authorized_keys examples for security. + - borg prune only knows "--keep-within" and not "--within" + - add gource video to resources docs, #507 + - authors: make it more clear what refers to borg and what to attic + - document standalone binary requirements, #499 + - rephrase the mailing list section + - development docs: run build_api and build_usage before tagging release + - internals docs: hash table max. load factor is 0.75 now + - markup, typo, grammar, phrasing, clarifications and other fixes. + + Version 0.29.0 -------------- From cf0262c8b4281556d3a30505cee18c793056036a Mon Sep 17 00:00:00 2001 From: Profpatsch Date: Mon, 18 Jan 2016 18:12:44 +0100 Subject: [PATCH 248/321] Fix upgrade without ~/attic/keys existing. fixes #576 --- borg/upgrader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/borg/upgrader.py b/borg/upgrader.py index bb47429e..d836c57a 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -267,10 +267,12 @@ class AtticKeyfileKey(KeyfileKey): get_keys_dir = cls.get_keys_dir id = hexlify(repository.id).decode('ascii') keys_dir = get_keys_dir() + if not os.path.exists(keys_dir): + raise KeyfileNotFoundError(repository.path, keys_dir) for name in os.listdir(keys_dir): filename = os.path.join(keys_dir, name) with open(filename, 'r') as fd: line = fd.readline().strip() if line and line.startswith(cls.FILE_ID) and line[10:] == id: return filename - raise KeyfileNotFoundError(repository.path, get_keys_dir()) + raise KeyfileNotFoundError(repository.path, keys_dir) From 854215b7dd8cb993fdd2dc56de402987d2df4892 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 20 Jan 2016 01:00:35 +0100 Subject: [PATCH 249/321] mention pp: in CHANGES --- docs/changes.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 4054bca4..98be66d3 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -29,11 +29,14 @@ Bug fixes: New features: -- implement exclusions using regular expression patterns. -- support patterns for borg extract, #361 +- borg create: implement exclusions using regular expression patterns. +- borg create: implement inclusions using patterns. +- borg extract: support patterns, #361 - support different styles for patterns: - fnmatch (fm: prefix, default when omitted), like borg <= 0.29. + - path prefix (pp: prefix, for unifying borg create pp1 pp2 into the + patterns system), semantics like in borg <= 0.29 - regular expression (re:), new! - --progress option for borg upgrade (#291) and borg delete - update progress indication more often (e.g. within big files), #500 From 382b79212b0eb345bf7f8452389ca5165fb75f05 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Thu, 21 Jan 2016 14:24:32 +0100 Subject: [PATCH 250/321] Reformat pattern syntax descriptions as definition list There are already three different styles and a fourth will be added. A definition list is easier to navigate when trying to find the description of a specific style. --- borg/archiver.py | 51 +++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 62f00f50..a076f7fa 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -628,38 +628,41 @@ class Archiver: helptext = {} helptext['patterns'] = textwrap.dedent(''' - Exclusion patterns support three separate styles, fnmatch, regular + Exclusion patterns support four separate styles, fnmatch, shell, regular expressions and path prefixes. If followed by a colon (':') the first two characters of a pattern are used as a style selector. Explicit style selection is necessary when a non-default style is desired or when the desired pattern starts with two alphanumeric characters followed by a colon (i.e. `aa:something/*`). - `Fnmatch `_ patterns use - a variant of shell pattern syntax, with '*' matching any number of - characters, '?' matching any single character, '[...]' matching any single - character specified, including ranges, and '[!...]' matching any character - not specified. The style selector is `fm`. For the purpose of these patterns, - the path separator ('\\' for Windows and '/' on other systems) is not treated - specially. For a path to match a pattern, it must completely match from start - to end, or must match from the start to just before a path separator. Except - for the root path, paths will never end in the path separator when matching - is attempted. Thus, if a given pattern ends in a path separator, a '*' is - appended before matching is attempted. + `Fnmatch `_, selector `fm:` - Regular expressions similar to those found in Perl are supported with the - selection prefix `re:`. Unlike shell patterns regular expressions are not - required to match the complete path and any substring match is sufficient. It - is strongly recommended to anchor patterns to the start ('^'), to the end - ('$') or both. Path separators ('\\' for Windows and '/' on other systems) in - paths are always normalized to a forward slash ('/') before applying - a pattern. The regular expression syntax is described in the `Python - documentation for the re module - `_. + These patterns use a variant of shell pattern syntax, with '*' matching + any number of characters, '?' matching any single character, '[...]' + matching any single character specified, including ranges, and '[!...]' + matching any character not specified. For the purpose of these patterns, + the path separator ('\\' for Windows and '/' on other systems) is not + treated specially. For a path to match a pattern, it must completely + match from start to end, or must match from the start to just before + a path separator. Except for the root path, paths will never end in the + path separator when matching is attempted. Thus, if a given pattern ends + in a path separator, a '*' is appended before matching is attempted. - Prefix path patterns can be selected with the prefix `pp:`. This pattern - style is useful to match whole sub-directories. The pattern `pp:/data/bar` - matches `/data/bar` and everything therein. + Regular expressions, selector `re:` + + Regular expressions similar to those found in Perl are supported. Unlike + shell patterns regular expressions are not required to match the complete + path and any substring match is sufficient. It is strongly recommended to + anchor patterns to the start ('^'), to the end ('$') or both. Path + separators ('\\' for Windows and '/' on other systems) in paths are + always normalized to a forward slash ('/') before applying a pattern. The + regular expression syntax is described in the `Python documentation for + the re module `_. + + Prefix path, selector `pp:` + + This pattern style is useful to match whole sub-directories. The pattern + `pp:/data/bar` matches `/data/bar` and everything therein. Exclusions can be passed via the command line option `--exclude`. When used from within a shell the patterns should be quoted to protect them from From c7fb598ab9d25701fd87dfcb568bd2beafc69616 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Thu, 21 Jan 2016 14:24:35 +0100 Subject: [PATCH 251/321] Add shell-style pattern syntax MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fnmatch module in Python's standard library implements a pattern format for paths which is similar to shell patterns. However, “*” matches any character including path separators. This newly introduced pattern syntax with the selector “sh” no longer matches the path separator with “*”. Instead “**/” can be used to match zero or more directory levels. --- borg/archiver.py | 20 ++++-- borg/helpers.py | 31 +++++++-- borg/shellpattern.py | 62 ++++++++++++++++++ borg/testsuite/helpers.py | 49 +++++++++++++- borg/testsuite/shellpattern.py | 113 +++++++++++++++++++++++++++++++++ docs/usage.rst | 4 ++ 6 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 borg/shellpattern.py create mode 100644 borg/testsuite/shellpattern.py diff --git a/borg/archiver.py b/borg/archiver.py index a076f7fa..8005d73b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -642,11 +642,20 @@ class Archiver: matching any single character specified, including ranges, and '[!...]' matching any character not specified. For the purpose of these patterns, the path separator ('\\' for Windows and '/' on other systems) is not - treated specially. For a path to match a pattern, it must completely - match from start to end, or must match from the start to just before - a path separator. Except for the root path, paths will never end in the - path separator when matching is attempted. Thus, if a given pattern ends - in a path separator, a '*' is appended before matching is attempted. + treated specially. Wrap meta-characters in brackets for a literal match + (i.e. `[?]` to match the literal character `?`). For a path to match + a pattern, it must completely match from start to end, or must match from + the start to just before a path separator. Except for the root path, + paths will never end in the path separator when matching is attempted. + Thus, if a given pattern ends in a path separator, a '*' is appended + before matching is attempted. + + Shell-style patterns, selector `sh:` + + Like fnmatch patterns these are similar to shell patterns. The difference + is that the pattern may include `**/` for matching zero or more directory + levels, `*` for matching zero or more arbitrary characters with the + exception of any path separator. Regular expressions, selector `re:` @@ -701,6 +710,7 @@ class Archiver: *.tmp fm:aa:something/* re:^/home/[^/]\.tmp/ + sh:/home/*/.thumbnails EOF $ borg create --exclude-from exclude.txt backup / ''') diff --git a/borg/helpers.py b/borg/helpers.py index 91c9e043..764d1dc4 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -30,6 +30,7 @@ from . import __version__ as borg_version from . import hashindex from . import chunker from . import crypto +from . import shellpattern import msgpack import msgpack.fallback @@ -332,11 +333,9 @@ class PatternBase: raise NotImplementedError -# For both PathPrefixPattern and FnmatchPattern, we require that -# the pattern either match the whole path or an initial segment -# of the path up to but not including a path separator. To -# unify the two cases, we add a path separator to the end of -# the path before matching. +# For PathPrefixPattern, FnmatchPattern and ShellPattern, we require that the pattern either match the whole path +# or an initial segment of the path up to but not including a path separator. To unify the two cases, we add a path +# separator to the end of the path before matching. class PathPrefixPattern(PatternBase): @@ -376,6 +375,27 @@ class FnmatchPattern(PatternBase): return (self.regex.match(path + os.path.sep) is not None) +class ShellPattern(PatternBase): + """Shell glob patterns to exclude. A trailing slash means to + exclude the contents of a directory, but not the directory itself. + """ + PREFIX = "sh" + + def _prepare(self, pattern): + sep = os.path.sep + + if pattern.endswith(sep): + pattern = os.path.normpath(pattern).rstrip(sep) + sep + "**" + sep + "*" + sep + else: + pattern = os.path.normpath(pattern) + sep + "**" + sep + "*" + + self.pattern = pattern + self.regex = re.compile(shellpattern.translate(self.pattern)) + + def _match(self, path): + return (self.regex.match(path + os.path.sep) is not None) + + class RegexPattern(PatternBase): """Regular expression to exclude. """ @@ -397,6 +417,7 @@ _PATTERN_STYLES = set([ FnmatchPattern, PathPrefixPattern, RegexPattern, + ShellPattern, ]) _PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES) diff --git a/borg/shellpattern.py b/borg/shellpattern.py new file mode 100644 index 00000000..7cb8f211 --- /dev/null +++ b/borg/shellpattern.py @@ -0,0 +1,62 @@ +import re +import os + + +def translate(pat): + """Translate a shell-style pattern to a regular expression. + + The pattern may include "**" ( stands for the platform-specific path separator; "/" on POSIX systems) for + matching zero or more directory levels and "*" for matching zero or more arbitrary characters with the exception of + any path separator. Wrap meta-characters in brackets for a literal match (i.e. "[?]" to match the literal character + "?"). + + This function is derived from the "fnmatch" module distributed with the Python standard library. + + Copyright (C) 2001-2016 Python Software Foundation. All rights reserved. + + TODO: support {alt1,alt2} shell-style alternatives + + """ + sep = os.path.sep + n = len(pat) + i = 0 + res = "" + + while i < n: + c = pat[i] + i += 1 + + if c == "*": + if i + 1 < n and pat[i] == "*" and pat[i + 1] == sep: + # **/ == wildcard for 0+ full (relative) directory names with trailing slashes; the forward slash stands + # for the platform-specific path separator + res += r"(?:[^\%s]*\%s)*" % (sep, sep) + i += 2 + else: + # * == wildcard for name parts (does not cross path separator) + res += r"[^\%s]*" % sep + elif c == "?": + # ? == any single character excluding path separator + res += r"[^\%s]" % sep + elif c == "[": + j = i + if j < n and pat[j] == "!": + j += 1 + if j < n and pat[j] == "]": + j += 1 + while j < n and pat[j] != "]": + j += 1 + if j >= n: + res += "\\[" + else: + stuff = pat[i:j].replace("\\", "\\\\") + i = j + 1 + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] == "^": + stuff = "\\" + stuff + res += "[%s]" % stuff + else: + res += re.escape(c) + + return res + r"\Z(?ms)" diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index f31bd984..cc4b3df3 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -12,7 +12,8 @@ import msgpack.fallback from ..helpers import Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher + ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher, \ + ShellPattern from . import BaseTestCase, environment_variable, FakeInputs @@ -234,6 +235,45 @@ def test_patterns_fnmatch(pattern, expected): check_patterns(files, FnmatchPattern(pattern), expected) +@pytest.mark.parametrize("pattern, expected", [ + # "None" means all files, i.e. all match the given pattern + ("*", None), + ("**/*", None), + ("/**/*", None), + ("/./*", None), + ("*/*", None), + ("*///*", None), + ("/home/u", []), + ("/home/*", + ["/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile", "/home/user2/public_html/index.html", + "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails"]), + ("/home/user/*", ["/home/user/.profile", "/home/user/.bashrc"]), + ("/etc/*/*", ["/etc/server/config", "/etc/server/hosts"]), + ("/etc/**/*", ["/etc/server/config", "/etc/server/hosts"]), + ("/etc/**/*/*", ["/etc/server/config", "/etc/server/hosts"]), + ("*/.pr????e", []), + ("**/.pr????e", ["/home/user/.profile", "/home/user2/.profile"]), + ("///etc//////*", ["/etc/server/config", "/etc/server/hosts"]), + ("/./home//..//home/user2/", ["/home/user2/.profile", "/home/user2/public_html/index.html"]), + ("/./home//..//home/user2/**/*", ["/home/user2/.profile", "/home/user2/public_html/index.html"]), + ("/srv*/", ["/srv/messages", "/srv/dmesg", "/srv2/blafasel"]), + ("/srv*", ["/srv", "/srv/messages", "/srv/dmesg", "/srv2", "/srv2/blafasel"]), + ("/srv/*", ["/srv/messages", "/srv/dmesg"]), + ("/srv2/**", ["/srv2", "/srv2/blafasel"]), + ("/srv2/**/", ["/srv2/blafasel"]), + ("/home/*/.thumbnails", ["/home/foo/.thumbnails"]), + ("/home/*/*/.thumbnails", ["/home/foo/bar/.thumbnails"]), + ]) +def test_patterns_shell(pattern, expected): + files = [ + "/etc/server/config", "/etc/server/hosts", "/home", "/home/user/.profile", "/home/user/.bashrc", + "/home/user2/.profile", "/home/user2/public_html/index.html", "/srv", "/srv/messages", "/srv/dmesg", + "/srv2", "/srv2/blafasel", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", + ] + + check_patterns(files, ShellPattern(pattern), expected) + + @pytest.mark.parametrize("pattern, expected", [ # "None" means all files, i.e. all match the given pattern ("", None), @@ -276,6 +316,7 @@ def _make_test_patterns(pattern): return [PathPrefixPattern(pattern), FnmatchPattern(pattern), RegexPattern("^{}/foo$".format(pattern)), + ShellPattern(pattern), ] @@ -374,6 +415,12 @@ def test_patterns_from_file(tmpdir, lines, expected): ("pp:/", PathPrefixPattern), ("pp:/data/", PathPrefixPattern), ("pp:pp:/data/", PathPrefixPattern), + + # Shell-pattern style + ("sh:", ShellPattern), + ("sh:*", ShellPattern), + ("sh:/data/*", ShellPattern), + ("sh:sh:/data/*", ShellPattern), ]) def test_parse_pattern(pattern, cls): assert isinstance(parse_pattern(pattern), cls) diff --git a/borg/testsuite/shellpattern.py b/borg/testsuite/shellpattern.py new file mode 100644 index 00000000..fae8c75d --- /dev/null +++ b/borg/testsuite/shellpattern.py @@ -0,0 +1,113 @@ +import re + +import pytest + +from .. import shellpattern + + +def check(path, pattern): + compiled = re.compile(shellpattern.translate(pattern)) + + return bool(compiled.match(path)) + + +@pytest.mark.parametrize("path, patterns", [ + # Literal string + ("foo/bar", ["foo/bar"]), + ("foo\\bar", ["foo\\bar"]), + + # Non-ASCII + ("foo/c/\u0152/e/bar", ["foo/*/\u0152/*/bar", "*/*/\u0152/*/*", "**/\u0152/*/*"]), + ("\u00e4\u00f6\u00dc", ["???", "*", "\u00e4\u00f6\u00dc", "[\u00e4][\u00f6][\u00dc]"]), + + # Question mark + ("foo", ["fo?"]), + ("foo", ["f?o"]), + ("foo", ["f??"]), + ("foo", ["?oo"]), + ("foo", ["?o?"]), + ("foo", ["??o"]), + ("foo", ["???"]), + + # Single asterisk + ("", ["*"]), + ("foo", ["*", "**", "***"]), + ("foo", ["foo*"]), + ("foobar", ["foo*"]), + ("foobar", ["foo*bar"]), + ("foobarbaz", ["foo*baz"]), + ("bar", ["*bar"]), + ("foobar", ["*bar"]), + ("foo/bar", ["foo/*bar"]), + ("foo/bar", ["foo/*ar"]), + ("foo/bar", ["foo/*r"]), + ("foo/bar", ["foo/*"]), + ("foo/bar", ["foo*/bar"]), + ("foo/bar", ["fo*/bar"]), + ("foo/bar", ["f*/bar"]), + ("foo/bar", ["*/bar"]), + + # Double asterisk (matches 0..n directory layers) + ("foo/bar", ["foo/**/bar"]), + ("foo/1/bar", ["foo/**/bar"]), + ("foo/1/22/333/bar", ["foo/**/bar"]), + ("foo/", ["foo/**/"]), + ("foo/1/", ["foo/**/"]), + ("foo/1/22/333/", ["foo/**/"]), + ("bar", ["**/bar"]), + ("1/bar", ["**/bar"]), + ("1/22/333/bar", ["**/bar"]), + ("foo/bar/baz", ["foo/**/*"]), + + # Set + ("foo1", ["foo[12]"]), + ("foo2", ["foo[12]"]), + ("foo2/bar", ["foo[12]/*"]), + ("f??f", ["f??f", "f[?][?]f"]), + ("foo]", ["foo[]]"]), + + # Inverted set + ("foo3", ["foo[!12]"]), + ("foo^", ["foo[^!]"]), + ("foo!", ["foo[^!]"]), + ]) +def test_match(path, patterns): + for p in patterns: + assert check(path, p) + + +@pytest.mark.parametrize("path, patterns", [ + ("", ["?", "[]"]), + ("foo", ["foo?"]), + ("foo", ["?foo"]), + ("foo", ["f?oo"]), + + # do not match path separator + ("foo/ar", ["foo?ar"]), + + # do not match/cross over os.path.sep + ("foo/bar", ["*"]), + ("foo/bar", ["foo*bar"]), + ("foo/bar", ["foo*ar"]), + ("foo/bar", ["fo*bar"]), + ("foo/bar", ["fo*ar"]), + + # Double asterisk + ("foobar", ["foo/**/bar"]), + + # Two asterisks without slash do not match directory separator + ("foo/bar", ["**"]), + + # Double asterisk not matching filename + ("foo/bar", ["**/"]), + + # Set + ("foo3", ["foo[12]"]), + + # Inverted set + ("foo1", ["foo[!12]"]), + ("foo2", ["foo[!12]"]), + ]) +def test_mismatch(path, patterns): + for p in patterns: + assert not check(path, p) diff --git a/docs/usage.rst b/docs/usage.rst index e3a9ed7b..894b1d96 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -240,6 +240,10 @@ Examples $ borg create /mnt/backup::my-files /home \ --exclude 're:^/home/[^/]+/\.thumbnails/' + # Do the same using a shell-style pattern + $ borg create /mnt/backup::my-files /home \ + --exclude 'sh:/home/*/.thumbnails' + # Backup the root filesystem into an archive named "root-YYYY-MM-DD" # use zlib compression (good, but slow) - default is no compression NAME="root-`date +%Y-%m-%d`" From 92969ea5f19bb4411182166a23489f1ed9580dcc Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Jan 2016 14:03:48 +0100 Subject: [PATCH 252/321] add gcc gcc-c++ to redhat/fedora/corora install docs --- docs/installation.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/installation.rst b/docs/installation.rst index 9ab397ee..6f088c91 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -139,6 +139,7 @@ Install the dependencies with development headers:: sudo dnf install openssl-devel openssl sudo dnf install libacl-devel libacl sudo dnf install lz4-devel + sudo dnf install gcc gcc-c++ sudo dnf install fuse-devel fuse pkgconfig # optional, for FUSE support From da7bc4af94d723ad6baa58c181afa578a62e6178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 22 Jan 2016 14:35:53 -0500 Subject: [PATCH 253/321] add netbsd install instructions --- docs/installation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/installation.rst b/docs/installation.rst index 6f088c91..38b1a9ce 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -34,6 +34,7 @@ Distribution Source Command ============ ============================================= ======= Arch Linux `[community]`_ ``pacman -S borg`` Debian `stretch`_, `unstable/sid`_ ``apt install borgbackup`` +NetBSD `pkgsrc`_ ``pkg_add py-borgbackup`` NixOS `.nix file`_ N/A OS X `Brew cask`_ ``brew cask install borgbackup`` Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` @@ -44,6 +45,7 @@ Ubuntu `Trusty 14.04 (backport PPA)`_ ``apt install borgbac .. _[community]: https://www.archlinux.org/packages/?name=borg .. _stretch: https://packages.debian.org/stretch/borgbackup .. _unstable/sid: https://packages.debian.org/sid/borgbackup +.. _pkgsrc: http://pkgsrc.se/sysutils/py-borgbackup .. _Xenial 16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup .. _Wily 15.10 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup .. _Vivid 15.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup From 69c8edc4e36195f3805c149fa576326ba525cbe5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Jan 2016 22:14:41 +0100 Subject: [PATCH 254/321] require virtualenv<14.0 so we get a py32 compatible pip --- .travis/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis/install.sh b/.travis/install.sh index 4de5e0c1..2e256e14 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -36,9 +36,9 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then ;; esac pyenv rehash - python -m pip install --user virtualenv + python -m pip install --user 'virtualenv<14.0' else - pip install virtualenv + pip install 'virtualenv<14.0' sudo add-apt-repository -y ppa:gezakovacs/lz4 sudo apt-get update sudo apt-get install -y liblz4-dev From a237c1fb5374b4bdc8b7bdd72bb0833f35e4d600 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 22 Jan 2016 23:12:08 +0100 Subject: [PATCH 255/321] add virtualenv<14.0 to requirements so tox does not pull in a py32 incompatible one --- requirements.d/development.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index dd9c3683..bf21d52a 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -1,3 +1,4 @@ +virtualenv<14.0 tox mock pytest From 14934dab08099d8adc76aa258349621f1181a831 Mon Sep 17 00:00:00 2001 From: Michael Hanselmann Date: Sat, 23 Jan 2016 11:05:06 +0100 Subject: [PATCH 256/321] Update changes for shell-style pattern support --- docs/changes.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 98be66d3..35aec31a 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -34,10 +34,12 @@ New features: - borg extract: support patterns, #361 - support different styles for patterns: - - fnmatch (fm: prefix, default when omitted), like borg <= 0.29. - - path prefix (pp: prefix, for unifying borg create pp1 pp2 into the + - fnmatch (`fm:` prefix, default when omitted), like borg <= 0.29. + - shell (`sh:` prefix) with `*` not matching directory separators and + `**/` matching 0..n directories + - path prefix (`pp:` prefix, for unifying borg create pp1 pp2 into the patterns system), semantics like in borg <= 0.29 - - regular expression (re:), new! + - regular expression (`re:`), new! - --progress option for borg upgrade (#291) and borg delete - update progress indication more often (e.g. within big files), #500 - finer chunker granularity for items metadata stream, #547, #487 From 541bbd4a5ba1deab6623d36c6c8da3d616049d85 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 15:21:38 +0100 Subject: [PATCH 257/321] add --list requirement hint --- docs/changes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 35aec31a..f1f33a42 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -9,6 +9,8 @@ Compatibility notes: - you may need to use -v (or --info) more often to actually see output emitted at INFO log level (because it is suppressed at the default WARNING log level). See the "general" section in the usage docs. +- for borg create, you need --list (additionally to -v) to see the long file + list (was needed so you can have e.g. --stats alone without the long list) - see below about BORG_DELETE_I_KNOW_WHAT_I_AM_DOING (was: BORG_CHECK_I_KNOW_WHAT_I_AM_DOING) From e2f5983eefd2ad923881f28b01840d2342c22a37 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 20:00:03 +0100 Subject: [PATCH 258/321] finer repo check progress indicator a step size of 5% was way too much, now doing 0.1% --- borg/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/repository.py b/borg/repository.py index 36cd0f66..80bbe669 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -346,7 +346,7 @@ class Repository: segments_transaction_id = self.io.get_segments_transaction_id() self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! segment_count = sum(1 for _ in self.io.segment_iterator()) - pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.0f%%", same_line=True) + pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.1f%%", step=0.1, same_line=True) for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) if segment > transaction_id: From 162d94b2e492eea2cf2c64681cf1ad798e5eb0db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 20:52:04 +0100 Subject: [PATCH 259/321] ran setup.py build_api --- docs/api.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 4bc7c763..628d21d1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -26,6 +26,10 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.shellpattern + :members: + :undoc-members: + .. automodule:: borg.repository :members: :undoc-members: @@ -38,6 +42,10 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.hash_sizes + :members: + :undoc-members: + .. automodule:: borg.xattr :members: :undoc-members: From dee1d462cc0f8673eb29df767b6b344232830f12 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 20:54:20 +0100 Subject: [PATCH 260/321] ran setup.py build_usage --- docs/usage/create.rst.inc | 3 +- docs/usage/delete.rst.inc | 5 +- docs/usage/extract.rst.inc | 2 +- docs/usage/help.rst.inc | 102 ++++++++++++++++++++------ docs/usage/migrate-to-repokey.rst.inc | 48 ++++++++++++ docs/usage/upgrade.rst.inc | 31 ++++---- 6 files changed, 150 insertions(+), 41 deletions(-) create mode 100644 docs/usage/migrate-to-repokey.rst.inc diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 96460f2a..273ba969 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -6,7 +6,7 @@ borg create usage: borg create [-h] [-v] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [-s] - [-p] [--filter STATUSCHARS] [-e PATTERN] + [-p] [--list] [--filter STATUSCHARS] [-e PATTERN] [--exclude-from EXCLUDEFILE] [--exclude-caches] [--exclude-if-present FILENAME] [--keep-tag-files] [-c SECONDS] [-x] [--numeric-owner] @@ -39,6 +39,7 @@ borg create showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen and the path being processed, default: False + --list output verbose list of items (files, dirs, ...) --filter STATUSCHARS only display items with the given status characters -e PATTERN, --exclude PATTERN exclude paths matching PATTERN diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index 1c7642b9..a278cc92 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -5,8 +5,8 @@ borg delete :: usage: borg delete [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-s] - [-c] [--save-space] + [--no-files-cache] [--umask M] [--remote-path PATH] [-p] + [-s] [-c] [--save-space] [TARGET] Delete an existing repository or archive @@ -26,6 +26,7 @@ borg delete detect unchanged files --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") + -p, --progress show progress display while deleting a single archive -s, --stats print statistics for the deleted archive -c, --cache-only delete only the local cache for the given repository --save-space work slower, but using less space diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index 62c45d46..9f2924fc 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -15,7 +15,7 @@ borg extract positional arguments: ARCHIVE archive to extract - PATH paths to extract + PATH paths to extract; patterns are supported optional arguments: -h, --help show this help message and exit diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 0379cd49..b7ea093b 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -5,30 +5,88 @@ borg help patterns :: - Exclude patterns use a variant of shell pattern syntax, with '*' matching any - number of characters, '?' matching any single character, '[...]' matching any - single character specified, including ranges, and '[!...]' matching any - character not specified. For the purpose of these patterns, the path - separator ('\' for Windows and '/' on other systems) is not treated - specially. For a path to match a pattern, it must completely match from - start to end, or must match from the start to just before a path separator. - Except for the root path, paths will never end in the path separator when - matching is attempted. Thus, if a given pattern ends in a path separator, a - '*' is appended before matching is attempted. Patterns with wildcards should - be quoted to protect them from shell expansion. +Exclusion patterns support four separate styles, fnmatch, shell, regular +expressions and path prefixes. If followed by a colon (':') the first two +characters of a pattern are used as a style selector. Explicit style +selection is necessary when a non-default style is desired or when the +desired pattern starts with two alphanumeric characters followed by a colon +(i.e. `aa:something/*`). - Examples: +`Fnmatch `_, selector `fm:` - # Exclude '/home/user/file.o' but not '/home/user/file.odt': - $ borg create -e '*.o' backup / + These patterns use a variant of shell pattern syntax, with '*' matching + any number of characters, '?' matching any single character, '[...]' + matching any single character specified, including ranges, and '[!...]' + matching any character not specified. For the purpose of these patterns, + the path separator ('\' for Windows and '/' on other systems) is not + treated specially. Wrap meta-characters in brackets for a literal match + (i.e. `[?]` to match the literal character `?`). For a path to match + a pattern, it must completely match from start to end, or must match from + the start to just before a path separator. Except for the root path, + paths will never end in the path separator when matching is attempted. + Thus, if a given pattern ends in a path separator, a '*' is appended + before matching is attempted. - # Exclude '/home/user/junk' and '/home/user/subdir/junk' but - # not '/home/user/importantjunk' or '/etc/junk': - $ borg create -e '/home/*/junk' backup / +Shell-style patterns, selector `sh:` - # Exclude the contents of '/home/user/cache' but not the directory itself: - $ borg create -e /home/user/cache/ backup / + Like fnmatch patterns these are similar to shell patterns. The difference + is that the pattern may include `**/` for matching zero or more directory + levels, `*` for matching zero or more arbitrary characters with the + exception of any path separator. - # The file '/home/user/cache/important' is *not* backed up: - $ borg create -e /home/user/cache/ backup / /home/user/cache/important - \ No newline at end of file +Regular expressions, selector `re:` + + Regular expressions similar to those found in Perl are supported. Unlike + shell patterns regular expressions are not required to match the complete + path and any substring match is sufficient. It is strongly recommended to + anchor patterns to the start ('^'), to the end ('$') or both. Path + separators ('\' for Windows and '/' on other systems) in paths are + always normalized to a forward slash ('/') before applying a pattern. The + regular expression syntax is described in the `Python documentation for + the re module `_. + +Prefix path, selector `pp:` + + This pattern style is useful to match whole sub-directories. The pattern + `pp:/data/bar` matches `/data/bar` and everything therein. + +Exclusions can be passed via the command line option `--exclude`. When used +from within a shell the patterns should be quoted to protect them from +expansion. + +The `--exclude-from` option permits loading exclusion patterns from a text +file with one pattern per line. Lines empty or starting with the number sign +('#') after removing whitespace on both ends are ignored. The optional style +selector prefix is also supported for patterns loaded from a file. Due to +whitespace removal paths with whitespace at the beginning or end can only be +excluded using regular expressions. + +Examples: + +# Exclude '/home/user/file.o' but not '/home/user/file.odt': +$ borg create -e '*.o' backup / + +# Exclude '/home/user/junk' and '/home/user/subdir/junk' but +# not '/home/user/importantjunk' or '/etc/junk': +$ borg create -e '/home/*/junk' backup / + +# Exclude the contents of '/home/user/cache' but not the directory itself: +$ borg create -e /home/user/cache/ backup / + +# The file '/home/user/cache/important' is *not* backed up: +$ borg create -e /home/user/cache/ backup / /home/user/cache/important + +# The contents of directories in '/home' are not backed up when their name +# ends in '.tmp' +$ borg create --exclude 're:^/home/[^/]+\.tmp/' backup / + +# Load exclusions from file +$ cat >exclude.txt < repokey + + positional arguments: + REPOSITORY + + optional arguments: + -h, --help show this help message and exit + -v, --verbose, --info + enable informative (verbose) output, work on log level + INFO + --debug enable debug output, work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + +Description +~~~~~~~~~~~ + +This command migrates a repository from passphrase mode (not supported any +more) to repokey mode. + +You will be first asked for the repository passphrase (to open it in passphrase +mode). This is the same passphrase as you used to use for this repo before 1.0. + +It will then derive the different secrets from this passphrase. + +Then you will be asked for a new passphrase (twice, for safety). This +passphrase will be used to protect the repokey (which contains these same +secrets in encrypted form). You may use the same passphrase as you used to +use, but you may also use a different one. + +After migrating to repokey mode, you can change the passphrase at any time. +But please note: the secrets will always stay the same and they could always +be derived from your (old) passphrase-mode passphrase. diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index 5aadcd3e..a630de13 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -5,8 +5,8 @@ borg upgrade :: usage: borg upgrade [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-n] - [-i] + [--no-files-cache] [--umask M] [--remote-path PATH] [-p] + [-n] [-i] [REPOSITORY] upgrade a repository from a previous version @@ -26,6 +26,7 @@ borg upgrade detect unchanged files --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") + -p, --progress show progress display while upgrading the repository -n, --dry-run do not change repository -i, --inplace rewrite repository in place, with no chance of going back to older versions of the repository. @@ -33,38 +34,38 @@ borg upgrade Description ~~~~~~~~~~~ -upgrade an existing Borg repository. this currently -only support converting an Attic repository, but may +Upgrade an existing Borg repository. This currently +only supports converting an Attic repository, but may eventually be extended to cover major Borg upgrades as well. -it will change the magic strings in the repository's segments -to match the new Borg magic strings. the keyfiles found in +It will change the magic strings in the repository's segments +to match the new Borg magic strings. The keyfiles found in $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and copied to $BORG_KEYS_DIR or ~/.borg/keys. -the cache files are converted, from $ATTIC_CACHE_DIR or +The cache files are converted, from $ATTIC_CACHE_DIR or ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the cache layout between Borg and Attic changed, so it is possible the first backup after the conversion takes longer than expected due to the cache resync. -upgrade should be able to resume if interrupted, although it -will still iterate over all segments. if you want to start +Upgrade should be able to resume if interrupted, although it +will still iterate over all segments. If you want to start from scratch, use `borg delete` over the copied repository to make sure the cache files are also removed: borg delete borg -unless ``--inplace`` is specified, the upgrade process first +Unless ``--inplace`` is specified, the upgrade process first creates a backup copy of the repository, in -REPOSITORY.upgrade-DATETIME, using hardlinks. this takes +REPOSITORY.upgrade-DATETIME, using hardlinks. This takes longer than in place upgrades, but is much safer and gives -progress information (as opposed to ``cp -al``). once you are +progress information (as opposed to ``cp -al``). Once you are satisfied with the conversion, you can safely destroy the backup copy. -WARNING: running the upgrade in place will make the current +WARNING: Running the upgrade in place will make the current copy unusable with older version, with no way of going back -to previous versions. this can PERMANENTLY DAMAGE YOUR +to previous versions. This can PERMANENTLY DAMAGE YOUR REPOSITORY! Attic CAN NOT READ BORG REPOSITORIES, as the -magic strings have changed. you have been warned. \ No newline at end of file +magic strings have changed. You have been warned. \ No newline at end of file From f4561e813f10797cf9b358b9cce38f94da8aeb95 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 21:43:24 +0100 Subject: [PATCH 261/321] update CHANGES --- docs/changes.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index f1f33a42..3f1735b2 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,8 +1,8 @@ Changelog ========= -Version 0.30.0 (not released yet) ---------------------------------- +Version 0.30.0 +-------------- Compatibility notes: @@ -43,7 +43,8 @@ New features: patterns system), semantics like in borg <= 0.29 - regular expression (`re:`), new! - --progress option for borg upgrade (#291) and borg delete -- update progress indication more often (e.g. within big files), #500 +- update progress indication more often (e.g. for borg create within big + files or for borg check repo), #500 - finer chunker granularity for items metadata stream, #547, #487 - borg create --list now used (additionally to -v) to enable the verbose file list output @@ -55,6 +56,10 @@ Other changes: grows fast while small, grows slower when getting bigger, #527 - Vagrantfile: use pyinstaller 3.1 to build binaries, freebsd sqlite3 fix, fixes #569 +- no separate binaries for centos6 any more because the generic linux binaries + also work on centos6 (or in general: on systems with a slightly older glibc + than debian7 +- dev environment: require virtualenv<14.0 so we get a py32 compatible pip - docs: - important: clarify -v and log levels in usage -> general, please read! @@ -63,12 +68,14 @@ Other changes: - disable unneeded SSH features in authorized_keys examples for security. - borg prune only knows "--keep-within" and not "--within" - add gource video to resources docs, #507 + - add netbsd install instructions - authors: make it more clear what refers to borg and what to attic - document standalone binary requirements, #499 - rephrase the mailing list section - development docs: run build_api and build_usage before tagging release - internals docs: hash table max. load factor is 0.75 now - markup, typo, grammar, phrasing, clarifications and other fixes. + - add gcc gcc-c++ to redhat/fedora/corora install docs, fixes #583 Version 0.29.0 From 74a9e8d52da50b6f666843234a1a6f8f33720430 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 22:10:42 +0100 Subject: [PATCH 262/321] Vagrantfile: remove python 3.2, use older pip/venv for trusty pyenv installs latest virtualenv/pip that is not compatible with py 3.2 any more I did a local python 3.2 tox run - it works. --- Vagrantfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 165fe4fa..968d18a5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -28,8 +28,10 @@ def packages_debianoid # for building python: apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev # this way it works on older dists (like ubuntu 12.04) also: - easy_install3 pip - pip3 install virtualenv + # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as + # newer versions are not compatible with py 3.2 any more. + easy_install3 'pip<8.0' + pip3 install 'virtualenv<14.0' touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile EOF end @@ -158,7 +160,6 @@ end def install_pythons(boxname) return <<-EOF . ~/.bash_profile - pyenv install 3.2.2 # tests, 3.2(.0) and 3.2.1 deadlock, issue #221 pyenv install 3.3.0 # tests pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests @@ -249,7 +250,7 @@ def run_tests(boxname) . ../borg-env/bin/activate if which pyenv > /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.2.2 3.3.0 3.4.0 3.5.0 + pyenv global 3.3.0 3.4.0 3.5.0 fi # otherwise: just use the system python if which fakeroot > /dev/null; then From 9ea79d738e123dc124e7a3139578a0d36e050b4d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 22:57:26 +0100 Subject: [PATCH 263/321] add chunks.archive.d trick to FAQ --- docs/changes.rst | 1 + docs/faq.rst | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 3f1735b2..de669bc3 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -62,6 +62,7 @@ Other changes: - dev environment: require virtualenv<14.0 so we get a py32 compatible pip - docs: + - add space-saving chunks.archive.d trick to FAQ - important: clarify -v and log levels in usage -> general, please read! - sphinx configuration: create a simple man page from usage docs - add a repo server setup example diff --git a/docs/faq.rst b/docs/faq.rst index acd6d859..f31b978e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -111,6 +111,33 @@ into the repository. Yes, as an attacker with access to the remote server could delete (or otherwise make unavailable) all your backups. +The borg cache eats way too much disk space, what can I do? +----------------------------------------------------------- + +There is a temporary (but maybe long lived) hack to avoid using lots of disk +space for chunks.archive.d (see issue #235 for details): + + # this assumes you are working with the same user as the backup + cd ~/.cache/borg/ + rm -rf chunks.archive.d ; touch chunks.archive.d + +This deletes all the cached archive chunk indexes and replaces the directory +that kept them with a file, so borg won't be able to store anything "in" there +in future. + +This has some pros and cons, though: + +- much less disk space needs for ~/.cache/borg. +- chunk cache resyncs will be slower as it will have to transfer chunk usage + metadata for all archives from the repository (which might be slow if your + repo connection is slow) and it will also have to build the hashtables from + that data. + chunk cache resyncs happen e.g. if your repo was written to by another + machine (if you share same backup repo between multiple machines) or if + your local chunks cache was lost somehow. + +The long term plan to improve this is called "borgception", see ticket #474. + If a backup stops mid-way, does the already-backed-up data stay there? ---------------------------------------------------------------------- From 12c7ef1329cc4dfaac78c5ca41848a15abf84cf0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 23 Jan 2016 23:06:05 +0100 Subject: [PATCH 264/321] Vagrantfile: avoid pkg-config missing error msg on netbsd --- Vagrantfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 968d18a5..eac1ff0a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -129,7 +129,8 @@ def packages_netbsd ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/ touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..." mozilla-rootcerts install - # pkg_add fuse pkg-config # llfuse 0.41.1 supports netbsd, but is still buggy. + pkg_add pkg-config # avoids some "pkg-config missing" error msg, even without fuse + # pkg_add fuse # llfuse 0.41.1 supports netbsd, but is still buggy. # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space pkg_add python34 py34-setuptools ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python From 22efee3d2efe9e79c90f53363f2d1118d393dec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sat, 23 Jan 2016 20:49:12 -0500 Subject: [PATCH 265/321] disambiguate -p versus -P we now use -P for --prefix and -p for --progress. previously, the result of -p depended on the command: some were using it for --progress, some for --prefix. this was confusing and was making it impossible to both --progress and --prefix with on-letter options --progress is likely used more often and interactively, so it get the keystroke shortcut (lower "-p") --prefix is used more rarely / in scripts, but important/dangerous for prune, so it get the extra keystroke (higher "-P") If somebody used -p someprefix and does not fix that to -P, it will result in "no archive specified" or "unrecognized argument". So it will neither cause pruning to remove wrong data nor go unnoticed. Closes: #563 --- borg/archiver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 8005d73b..0bf6f597 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -862,7 +862,7 @@ class Archiver: subparser.add_argument('--last', dest='last', type=int, default=None, metavar='N', help='only check last N archives (Default: all)') - subparser.add_argument('-p', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') change_passphrase_epilog = textwrap.dedent(""" @@ -1048,7 +1048,7 @@ class Archiver: subparser.add_argument('--short', dest='short', action='store_true', default=False, help='only print file/directory names, nothing else') - subparser.add_argument('-p', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), @@ -1120,7 +1120,7 @@ class Archiver: "1m" is taken to mean "31d". The archives kept with this option do not count towards the totals specified by any other options. - If a prefix is set with -p, then only archives that start with the prefix are + If a prefix is set with -P, then only archives that start with the prefix are considered for deletion and only those archives count towards the totals specified by the rules. Otherwise, *all* archives in the repository are candidates for deletion! @@ -1148,7 +1148,7 @@ class Archiver: help='number of monthly archives to keep') subparser.add_argument('-y', '--keep-yearly', dest='yearly', type=int, default=0, help='number of yearly archives to keep') - subparser.add_argument('-p', '--prefix', dest='prefix', type=str, + subparser.add_argument('-P', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, From 777fc89d3f8eb0cd3a4a8f0b44e1938883ee71b2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 24 Jan 2016 14:49:07 +0100 Subject: [PATCH 266/321] Vagrantfile: rsync symlinks as symlinks, fixes #592 --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index eac1ff0a..28429854 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -273,7 +273,7 @@ end Vagrant.configure(2) do |config| # use rsync to copy content to the folder - config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync" + config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"] # do not let the VM access . on the host machine via the default shared folder! config.vm.synced_folder ".", "/vagrant", disabled: true From 7d591226d2c28756fc8024100511f6269281e820 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 21:02:02 +0100 Subject: [PATCH 267/321] remove support for python 3.2.x and 3.3.x, require 3.4+, fixes #65, fixes #221 --- .travis.yml | 14 -------------- .travis/install.sh | 12 ++---------- Vagrantfile | 3 +-- setup.py | 4 +--- tox.ini | 2 +- 5 files changed, 5 insertions(+), 30 deletions(-) diff --git a/.travis.yml b/.travis.yml index 156391f7..5b46928f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,26 +8,12 @@ cache: matrix: include: - - python: 3.2 - os: linux - env: TOXENV=py32 - - python: 3.3 - os: linux - env: TOXENV=py33 - python: 3.4 os: linux env: TOXENV=py34 - python: 3.5 os: linux env: TOXENV=py35 - - language: generic - os: osx - osx_image: xcode6.4 - env: TOXENV=py32 - - language: generic - os: osx - osx_image: xcode6.4 - env: TOXENV=py33 - language: generic os: osx osx_image: xcode6.4 diff --git a/.travis/install.sh b/.travis/install.sh index 2e256e14..73e292dd 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -18,21 +18,13 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then brew outdated pyenv || brew upgrade pyenv case "${TOXENV}" in - py32) - pyenv install 3.2.6 - pyenv global 3.2.6 - ;; - py33) - pyenv install 3.3.6 - pyenv global 3.3.6 - ;; py34) pyenv install 3.4.3 pyenv global 3.4.3 ;; py35) - pyenv install 3.5.0 - pyenv global 3.5.0 + pyenv install 3.5.1 + pyenv global 3.5.1 ;; esac pyenv rehash diff --git a/Vagrantfile b/Vagrantfile index 28429854..bfe0f764 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -161,7 +161,6 @@ end def install_pythons(boxname) return <<-EOF . ~/.bash_profile - pyenv install 3.3.0 # tests pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests pyenv install 3.5.1 # binary build, use latest 3.5.x release @@ -251,7 +250,7 @@ def run_tests(boxname) . ../borg-env/bin/activate if which pyenv > /dev/null; then # for testing, use the earliest point releases of the supported python versions: - pyenv global 3.3.0 3.4.0 3.5.0 + pyenv global 3.4.0 3.5.0 fi # otherwise: just use the system python if which fakeroot > /dev/null; then diff --git a/setup.py b/setup.py index c3ed123f..6a4d34bd 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ from glob import glob from distutils.command.build import build from distutils.core import Command -min_python = (3, 2) +min_python = (3, 4) my_python = sys.version_info if my_python < min_python: @@ -242,8 +242,6 @@ setup( 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Topic :: Security :: Cryptography', diff --git a/tox.ini b/tox.ini index a000e1af..bc9e6db8 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # fakeroot -u tox --recreate [tox] -envlist = py{32,33,34,35} +envlist = py{34,35} [testenv] # Change dir to avoid import problem for cython code. The directory does From a2843bc939fa3f57aec100f8e121af8677e869bf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 21:11:38 +0100 Subject: [PATCH 268/321] docs: require python 3.4+ remove references to older pythons. --- docs/development.rst | 2 +- docs/faq.rst | 2 +- docs/installation.rst | 2 +- docs/usage.rst | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 42036950..3a119218 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -60,7 +60,7 @@ Some more advanced examples:: # verify a changed tox.ini (run this after any change to tox.ini): fakeroot -u tox --recreate - fakeroot -u tox -e py32 # run all tests, but only on python 3.2 + fakeroot -u tox -e py34 # run all tests, but only on python 3.4 fakeroot -u tox borg.testsuite.locking # only run 1 test module diff --git a/docs/faq.rst b/docs/faq.rst index f31b978e..1831bee6 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -37,7 +37,7 @@ Which file types, attributes, etc. are preserved? * FIFOs ("named pipes") * Name * Contents - * Time of last modification (nanosecond precision with Python >= 3.3) + * Time of last modification (nanosecond precision) * IDs of owning user and owning group * Names of owning user and owning group (if the IDs can be resolved) * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) diff --git a/docs/installation.rst b/docs/installation.rst index 38b1a9ce..f4a963dd 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -98,7 +98,7 @@ Dependencies To install |project_name| from a source package (including pip), you have to install the following dependencies first: -* `Python 3`_ >= 3.2.2. Even though Python 3 is not the default Python version on +* `Python 3`_ >= 3.4.0. Even though Python 3 is not the default Python version on most systems, it is usually available as an optional install. * OpenSSL_ >= 1.0.0 * libacl_ (that pulls in libattr_ also) diff --git a/docs/usage.rst b/docs/usage.rst index 894b1d96..4abd44b2 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -100,7 +100,7 @@ Please note: (e.g. mode 600, root:root). -.. _INI: https://docs.python.org/3.2/library/logging.config.html#configuration-file-format +.. _INI: https://docs.python.org/3.4/library/logging.config.html#configuration-file-format Resource Usage ~~~~~~~~~~~~~~ From 8a819d4499bef2722ff6b3445f65878018f1d609 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 21:23:52 +0100 Subject: [PATCH 269/321] remove borg.support, fixes #358 we only needed it because argparse was broken on some 3.2.x and 3.3.x pythons. --- borg/archiver.py | 4 +- borg/helpers.py | 3 +- borg/support/__init__.py | 16 - borg/support/argparse.py | 2383 -------------------------------------- setup.py | 2 +- 5 files changed, 3 insertions(+), 2405 deletions(-) delete mode 100644 borg/support/__init__.py delete mode 100644 borg/support/argparse.py diff --git a/borg/archiver.py b/borg/archiver.py index 0bf6f597..ffb0727c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1,10 +1,8 @@ -from .support import argparse # see support/__init__.py docstring - # DEPRECATED - remove after requiring py 3.4 - from binascii import hexlify from datetime import datetime from hashlib import sha256 from operator import attrgetter +import argparse import functools import inspect import io diff --git a/borg/helpers.py b/borg/helpers.py index 764d1dc4..8670419b 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -1,5 +1,4 @@ -from .support import argparse # see support/__init__.py docstring, DEPRECATED - remove after requiring py 3.4 - +import argparse import binascii from collections import namedtuple from functools import wraps diff --git a/borg/support/__init__.py b/borg/support/__init__.py deleted file mode 100644 index 449fcebf..00000000 --- a/borg/support/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -3rd party stuff that needed fixing - -Note: linux package maintainers feel free to remove any of these hacks - IF your python version is not affected. - -argparse is broken with default args (double conversion): -affects: 3.2.0 <= python < 3.2.4 -affects: 3.3.0 <= python < 3.3.1 - -as we still support 3.2 and 3.3 there is no other way than to bundle -a fixed version (I just took argparse.py from 3.2.6) and import it from -here (see import in archiver.py). -DEPRECATED - remove support.argparse after requiring python 3.4. -""" - diff --git a/borg/support/argparse.py b/borg/support/argparse.py deleted file mode 100644 index da73bc5f..00000000 --- a/borg/support/argparse.py +++ /dev/null @@ -1,2383 +0,0 @@ -# Author: Steven J. Bethard . - -"""Command-line parsing library - -This module is an optparse-inspired command-line parsing library that: - - - handles both optional and positional arguments - - produces highly informative usage messages - - supports parsers that dispatch to sub-parsers - -The following is a simple usage example that sums integers from the -command-line and writes the result to a file:: - - parser = argparse.ArgumentParser( - description='sum the integers at the command line') - parser.add_argument( - 'integers', metavar='int', nargs='+', type=int, - help='an integer to be summed') - parser.add_argument( - '--log', default=sys.stdout, type=argparse.FileType('w'), - help='the file where the sum should be written') - args = parser.parse_args() - args.log.write('%s' % sum(args.integers)) - args.log.close() - -The module contains the following public classes: - - - ArgumentParser -- The main entry point for command-line parsing. As the - example above shows, the add_argument() method is used to populate - the parser with actions for optional and positional arguments. Then - the parse_args() method is invoked to convert the args at the - command-line into an object with attributes. - - - ArgumentError -- The exception raised by ArgumentParser objects when - there are errors with the parser's actions. Errors raised while - parsing the command-line are caught by ArgumentParser and emitted - as command-line messages. - - - FileType -- A factory for defining types of files to be created. As the - example above shows, instances of FileType are typically passed as - the type= argument of add_argument() calls. - - - Action -- The base class for parser actions. Typically actions are - selected by passing strings like 'store_true' or 'append_const' to - the action= argument of add_argument(). However, for greater - customization of ArgumentParser actions, subclasses of Action may - be defined and passed as the action= argument. - - - HelpFormatter, RawDescriptionHelpFormatter, RawTextHelpFormatter, - ArgumentDefaultsHelpFormatter -- Formatter classes which - may be passed as the formatter_class= argument to the - ArgumentParser constructor. HelpFormatter is the default, - RawDescriptionHelpFormatter and RawTextHelpFormatter tell the parser - not to change the formatting for help text, and - ArgumentDefaultsHelpFormatter adds information about argument defaults - to the help. - -All other classes in this module are considered implementation details. -(Also note that HelpFormatter and RawDescriptionHelpFormatter are only -considered public as object names -- the API of the formatter objects is -still considered an implementation detail.) -""" - -__version__ = '1.1' -__all__ = [ - 'ArgumentParser', - 'ArgumentError', - 'ArgumentTypeError', - 'FileType', - 'HelpFormatter', - 'ArgumentDefaultsHelpFormatter', - 'RawDescriptionHelpFormatter', - 'RawTextHelpFormatter', - 'Namespace', - 'Action', - 'ONE_OR_MORE', - 'OPTIONAL', - 'PARSER', - 'REMAINDER', - 'SUPPRESS', - 'ZERO_OR_MORE', -] - - -import collections as _collections -import copy as _copy -import os as _os -import re as _re -import sys as _sys -import textwrap as _textwrap - -try: - from gettext import gettext, ngettext -except ImportError: - def gettext(message): - return message - def ngettext(msg1, msg2, n): - return msg1 if n == 1 else msg2 -_ = gettext - - -SUPPRESS = '==SUPPRESS==' - -OPTIONAL = '?' -ZERO_OR_MORE = '*' -ONE_OR_MORE = '+' -PARSER = 'A...' -REMAINDER = '...' -_UNRECOGNIZED_ARGS_ATTR = '_unrecognized_args' - -# ============================= -# Utility functions and classes -# ============================= - -class _AttributeHolder(object): - """Abstract base class that provides __repr__. - - The __repr__ method returns a string in the format:: - ClassName(attr=name, attr=name, ...) - The attributes are determined either by a class-level attribute, - '_kwarg_names', or by inspecting the instance __dict__. - """ - - def __repr__(self): - type_name = type(self).__name__ - arg_strings = [] - for arg in self._get_args(): - arg_strings.append(repr(arg)) - for name, value in self._get_kwargs(): - arg_strings.append('%s=%r' % (name, value)) - return '%s(%s)' % (type_name, ', '.join(arg_strings)) - - def _get_kwargs(self): - return sorted(self.__dict__.items()) - - def _get_args(self): - return [] - - -def _ensure_value(namespace, name, value): - if getattr(namespace, name, None) is None: - setattr(namespace, name, value) - return getattr(namespace, name) - - -# =============== -# Formatting Help -# =============== - -class HelpFormatter(object): - """Formatter for generating usage messages and argument help strings. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def __init__(self, - prog, - indent_increment=2, - max_help_position=24, - width=None): - - # default setting for width - if width is None: - try: - width = int(_os.environ['COLUMNS']) - except (KeyError, ValueError): - width = 80 - width -= 2 - - self._prog = prog - self._indent_increment = indent_increment - self._max_help_position = max_help_position - self._width = width - - self._current_indent = 0 - self._level = 0 - self._action_max_length = 0 - - self._root_section = self._Section(self, None) - self._current_section = self._root_section - - self._whitespace_matcher = _re.compile(r'\s+') - self._long_break_matcher = _re.compile(r'\n\n\n+') - - # =============================== - # Section and indentation methods - # =============================== - def _indent(self): - self._current_indent += self._indent_increment - self._level += 1 - - def _dedent(self): - self._current_indent -= self._indent_increment - assert self._current_indent >= 0, 'Indent decreased below 0.' - self._level -= 1 - - class _Section(object): - - def __init__(self, formatter, parent, heading=None): - self.formatter = formatter - self.parent = parent - self.heading = heading - self.items = [] - - def format_help(self): - # format the indented section - if self.parent is not None: - self.formatter._indent() - join = self.formatter._join_parts - for func, args in self.items: - func(*args) - item_help = join([func(*args) for func, args in self.items]) - if self.parent is not None: - self.formatter._dedent() - - # return nothing if the section was empty - if not item_help: - return '' - - # add the heading if the section was non-empty - if self.heading is not SUPPRESS and self.heading is not None: - current_indent = self.formatter._current_indent - heading = '%*s%s:\n' % (current_indent, '', self.heading) - else: - heading = '' - - # join the section-initial newline, the heading and the help - return join(['\n', heading, item_help, '\n']) - - def _add_item(self, func, args): - self._current_section.items.append((func, args)) - - # ======================== - # Message building methods - # ======================== - def start_section(self, heading): - self._indent() - section = self._Section(self, self._current_section, heading) - self._add_item(section.format_help, []) - self._current_section = section - - def end_section(self): - self._current_section = self._current_section.parent - self._dedent() - - def add_text(self, text): - if text is not SUPPRESS and text is not None: - self._add_item(self._format_text, [text]) - - def add_usage(self, usage, actions, groups, prefix=None): - if usage is not SUPPRESS: - args = usage, actions, groups, prefix - self._add_item(self._format_usage, args) - - def add_argument(self, action): - if action.help is not SUPPRESS: - - # find all invocations - get_invocation = self._format_action_invocation - invocations = [get_invocation(action)] - for subaction in self._iter_indented_subactions(action): - invocations.append(get_invocation(subaction)) - - # update the maximum item length - invocation_length = max([len(s) for s in invocations]) - action_length = invocation_length + self._current_indent - self._action_max_length = max(self._action_max_length, - action_length) - - # add the item to the list - self._add_item(self._format_action, [action]) - - def add_arguments(self, actions): - for action in actions: - self.add_argument(action) - - # ======================= - # Help-formatting methods - # ======================= - def format_help(self): - help = self._root_section.format_help() - if help: - help = self._long_break_matcher.sub('\n\n', help) - help = help.strip('\n') + '\n' - return help - - def _join_parts(self, part_strings): - return ''.join([part - for part in part_strings - if part and part is not SUPPRESS]) - - def _format_usage(self, usage, actions, groups, prefix): - if prefix is None: - prefix = _('usage: ') - - # if usage is specified, use that - if usage is not None: - usage = usage % dict(prog=self._prog) - - # if no optionals or positionals are available, usage is just prog - elif usage is None and not actions: - usage = '%(prog)s' % dict(prog=self._prog) - - # if optionals and positionals are available, calculate usage - elif usage is None: - prog = '%(prog)s' % dict(prog=self._prog) - - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - - # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) - - # wrap the usage parts if it's too long - text_width = self._width - self._current_indent - if len(prefix) + len(usage) > text_width: - - # break usage into wrappable parts - part_regexp = r'\(.*?\)+|\[.*?\]+|\S+' - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage - - # helper for wrapping lines - def get_lines(parts, indent, prefix=None): - lines = [] - line = [] - if prefix is not None: - line_len = len(prefix) - 1 - else: - line_len = len(indent) - 1 - for part in parts: - if line_len + 1 + len(part) > text_width: - lines.append(indent + ' '.join(line)) - line = [] - line_len = len(indent) - 1 - line.append(part) - line_len += len(part) + 1 - if line: - lines.append(indent + ' '.join(line)) - if prefix is not None: - lines[0] = lines[0][len(indent):] - return lines - - # if prog is short, follow it with optionals or positionals - if len(prefix) + len(prog) <= 0.75 * text_width: - indent = ' ' * (len(prefix) + len(prog) + 1) - if opt_parts: - lines = get_lines([prog] + opt_parts, indent, prefix) - lines.extend(get_lines(pos_parts, indent)) - elif pos_parts: - lines = get_lines([prog] + pos_parts, indent, prefix) - else: - lines = [prog] - - # if prog is long, put it on its own line - else: - indent = ' ' * len(prefix) - parts = opt_parts + pos_parts - lines = get_lines(parts, indent) - if len(lines) > 1: - lines = [] - lines.extend(get_lines(opt_parts, indent)) - lines.extend(get_lines(pos_parts, indent)) - lines = [prog] + lines - - # join lines into usage - usage = '\n'.join(lines) - - # prefix with 'usage:' - return '%s%s\n\n' % (prefix, usage) - - def _format_actions_usage(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - try: - start = actions.index(group._group_actions[0]) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: - for action in group._group_actions: - group_actions.add(action) - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - inserts[end] = ']' - else: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' - - # collect all actions format strings - parts = [] - for i, action in enumerate(actions): - - # suppressed arguments are marked with None - # remove | separators for suppressed arguments - if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) - - # produce all arg strings - elif not action.option_strings: - part = self._format_args(action, action.dest) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # add the action string to the list - parts.append(part) - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = '%s' % option_string - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - part = '%s %s' % (option_string, args_string) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = _re.sub(r'\(([^|]*)\)', r'\1', text) - text = text.strip() - - # return the text - return text - - def _format_text(self, text): - if '%(prog)' in text: - text = text % dict(prog=self._prog) - text_width = self._width - self._current_indent - indent = ' ' * self._current_indent - return self._fill_text(text, text_width, indent) + '\n\n' - - def _format_action(self, action): - # determine the required width and the entry label - help_position = min(self._action_max_length + 2, - self._max_help_position) - help_width = self._width - help_position - action_width = help_position - self._current_indent - 2 - action_header = self._format_action_invocation(action) - - # ho nelp; start on same line and add a final newline - if not action.help: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - - # short action name; start on the same line and pad two spaces - elif len(action_header) <= action_width: - tup = self._current_indent, '', action_width, action_header - action_header = '%*s%-*s ' % tup - indent_first = 0 - - # long action name; start on the next line - else: - tup = self._current_indent, '', action_header - action_header = '%*s%s\n' % tup - indent_first = help_position - - # collect the pieces of the action help - parts = [action_header] - - # if there was help for the action, add lines of help text - if action.help: - help_text = self._expand_help(action) - help_lines = self._split_lines(help_text, help_width) - parts.append('%*s%s\n' % (indent_first, '', help_lines[0])) - for line in help_lines[1:]: - parts.append('%*s%s\n' % (help_position, '', line)) - - # or add a newline if the description doesn't end with one - elif not action_header.endswith('\n'): - parts.append('\n') - - # if there are any sub-actions, add their help as well - for subaction in self._iter_indented_subactions(action): - parts.append(self._format_action(subaction)) - - # return a single string - return self._join_parts(parts) - - def _format_action_invocation(self, action): - if not action.option_strings: - metavar, = self._metavar_formatter(action, action.dest)(1) - return metavar - - else: - parts = [] - - # if the Optional doesn't take a value, format is: - # -s, --long - if action.nargs == 0: - parts.extend(action.option_strings) - - # if the Optional takes a value, format is: - # -s ARGS, --long ARGS - else: - default = action.dest.upper() - args_string = self._format_args(action, default) - for option_string in action.option_strings: - parts.append('%s %s' % (option_string, args_string)) - - return ', '.join(parts) - - def _metavar_formatter(self, action, default_metavar): - if action.metavar is not None: - result = action.metavar - elif action.choices is not None: - choice_strs = [str(choice) for choice in action.choices] - result = '{%s}' % ','.join(choice_strs) - else: - result = default_metavar - - def format(tuple_size): - if isinstance(result, tuple): - return result - else: - return (result, ) * tuple_size - return format - - def _format_args(self, action, default_metavar): - get_metavar = self._metavar_formatter(action, default_metavar) - if action.nargs is None: - result = '%s' % get_metavar(1) - elif action.nargs == OPTIONAL: - result = '[%s]' % get_metavar(1) - elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % get_metavar(2) - elif action.nargs == ONE_OR_MORE: - result = '%s [%s ...]' % get_metavar(2) - elif action.nargs == REMAINDER: - result = '...' - elif action.nargs == PARSER: - result = '%s ...' % get_metavar(1) - else: - formats = ['%s' for _ in range(action.nargs)] - result = ' '.join(formats) % get_metavar(action.nargs) - return result - - def _expand_help(self, action): - params = dict(vars(action), prog=self._prog) - for name in list(params): - if params[name] is SUPPRESS: - del params[name] - for name in list(params): - if hasattr(params[name], '__name__'): - params[name] = params[name].__name__ - if params.get('choices') is not None: - choices_str = ', '.join([str(c) for c in params['choices']]) - params['choices'] = choices_str - return self._get_help_string(action) % params - - def _iter_indented_subactions(self, action): - try: - get_subactions = action._get_subactions - except AttributeError: - pass - else: - self._indent() - for subaction in get_subactions(): - yield subaction - self._dedent() - - def _split_lines(self, text, width): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.wrap(text, width) - - def _fill_text(self, text, width, indent): - text = self._whitespace_matcher.sub(' ', text).strip() - return _textwrap.fill(text, width, initial_indent=indent, - subsequent_indent=indent) - - def _get_help_string(self, action): - return action.help - - -class RawDescriptionHelpFormatter(HelpFormatter): - """Help message formatter which retains any formatting in descriptions. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _fill_text(self, text, width, indent): - return ''.join([indent + line for line in text.splitlines(True)]) - - -class RawTextHelpFormatter(RawDescriptionHelpFormatter): - """Help message formatter which retains formatting of all help text. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _split_lines(self, text, width): - return text.splitlines() - - -class ArgumentDefaultsHelpFormatter(HelpFormatter): - """Help message formatter which adds default values to argument help. - - Only the name of this class is considered a public API. All the methods - provided by the class are considered an implementation detail. - """ - - def _get_help_string(self, action): - help = action.help - if '%(default)' not in action.help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += ' (default: %(default)s)' - return help - - -# ===================== -# Options and Arguments -# ===================== - -def _get_action_name(argument): - if argument is None: - return None - elif argument.option_strings: - return '/'.join(argument.option_strings) - elif argument.metavar not in (None, SUPPRESS): - return argument.metavar - elif argument.dest not in (None, SUPPRESS): - return argument.dest - else: - return None - - -class ArgumentError(Exception): - """An error from creating or using an argument (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - """ - - def __init__(self, argument, message): - self.argument_name = _get_action_name(argument) - self.message = message - - def __str__(self): - if self.argument_name is None: - format = '%(message)s' - else: - format = 'argument %(argument_name)s: %(message)s' - return format % dict(message=self.message, - argument_name=self.argument_name) - - -class ArgumentTypeError(Exception): - """An error from trying to convert a command line string to a type.""" - pass - - -# ============== -# Action classes -# ============== - -class Action(_AttributeHolder): - """Information about how to convert command line strings to Python objects. - - Action objects are used by an ArgumentParser to represent the information - needed to parse a single argument from one or more strings from the - command line. The keyword arguments to the Action constructor are also - all attributes of Action instances. - - Keyword Arguments: - - - option_strings -- A list of command-line option strings which - should be associated with this action. - - - dest -- The name of the attribute to hold the created object(s) - - - nargs -- The number of command-line arguments that should be - consumed. By default, one argument will be consumed and a single - value will be produced. Other values include: - - N (an integer) consumes N arguments (and produces a list) - - '?' consumes zero or one arguments - - '*' consumes zero or more arguments (and produces a list) - - '+' consumes one or more arguments (and produces a list) - Note that the difference between the default and nargs=1 is that - with the default, a single value will be produced, while with - nargs=1, a list containing a single value will be produced. - - - const -- The value to be produced if the option is specified and the - option uses an action that takes no values. - - - default -- The value to be produced if the option is not specified. - - - type -- A callable that accepts a single string argument, and - returns the converted value. The standard Python types str, int, - float, and complex are useful examples of such callables. If None, - str is used. - - - choices -- A container of values that should be allowed. If not None, - after a command-line argument has been converted to the appropriate - type, an exception will be raised if it is not a member of this - collection. - - - required -- True if the action must always be specified at the - command line. This is only meaningful for optional command-line - arguments. - - - help -- The help string describing the argument. - - - metavar -- The name to be used for the option's argument with the - help string. If None, the 'dest' value will be used as the name. - """ - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - self.option_strings = option_strings - self.dest = dest - self.nargs = nargs - self.const = const - self.default = default - self.type = type - self.choices = choices - self.required = required - self.help = help - self.metavar = metavar - - def _get_kwargs(self): - names = [ - 'option_strings', - 'dest', - 'nargs', - 'const', - 'default', - 'type', - 'choices', - 'help', - 'metavar', - ] - return [(name, getattr(self, name)) for name in names] - - def __call__(self, parser, namespace, values, option_string=None): - raise NotImplementedError(_('.__call__() not defined')) - - -class _StoreAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for store actions must be > 0; if you ' - 'have nothing to store, actions such as store ' - 'true or store const may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_StoreAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, values) - - -class _StoreConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_StoreConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - setattr(namespace, self.dest, self.const) - - -class _StoreTrueAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=False, - required=False, - help=None): - super(_StoreTrueAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=True, - default=default, - required=required, - help=help) - - -class _StoreFalseAction(_StoreConstAction): - - def __init__(self, - option_strings, - dest, - default=True, - required=False, - help=None): - super(_StoreFalseAction, self).__init__( - option_strings=option_strings, - dest=dest, - const=False, - default=default, - required=required, - help=help) - - -class _AppendAction(Action): - - def __init__(self, - option_strings, - dest, - nargs=None, - const=None, - default=None, - type=None, - choices=None, - required=False, - help=None, - metavar=None): - if nargs == 0: - raise ValueError('nargs for append actions must be > 0; if arg ' - 'strings are not supplying the value to append, ' - 'the append const action may be more appropriate') - if const is not None and nargs != OPTIONAL: - raise ValueError('nargs must be %r to supply const' % OPTIONAL) - super(_AppendAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(values) - setattr(namespace, self.dest, items) - - -class _AppendConstAction(Action): - - def __init__(self, - option_strings, - dest, - const, - default=None, - required=False, - help=None, - metavar=None): - super(_AppendConstAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - const=const, - default=default, - required=required, - help=help, - metavar=metavar) - - def __call__(self, parser, namespace, values, option_string=None): - items = _copy.copy(_ensure_value(namespace, self.dest, [])) - items.append(self.const) - setattr(namespace, self.dest, items) - - -class _CountAction(Action): - - def __init__(self, - option_strings, - dest, - default=None, - required=False, - help=None): - super(_CountAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=0, - default=default, - required=required, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - new_count = _ensure_value(namespace, self.dest, 0) + 1 - setattr(namespace, self.dest, new_count) - - -class _HelpAction(Action): - - def __init__(self, - option_strings, - dest=SUPPRESS, - default=SUPPRESS, - help=None): - super(_HelpAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - - def __call__(self, parser, namespace, values, option_string=None): - parser.print_help() - parser.exit() - - -class _VersionAction(Action): - - def __init__(self, - option_strings, - version=None, - dest=SUPPRESS, - default=SUPPRESS, - help="show program's version number and exit"): - super(_VersionAction, self).__init__( - option_strings=option_strings, - dest=dest, - default=default, - nargs=0, - help=help) - self.version = version - - def __call__(self, parser, namespace, values, option_string=None): - version = self.version - if version is None: - version = parser.version - formatter = parser._get_formatter() - formatter.add_text(version) - parser.exit(message=formatter.format_help()) - - -class _SubParsersAction(Action): - - class _ChoicesPseudoAction(Action): - - def __init__(self, name, aliases, help): - metavar = dest = name - if aliases: - metavar += ' (%s)' % ', '.join(aliases) - sup = super(_SubParsersAction._ChoicesPseudoAction, self) - sup.__init__(option_strings=[], dest=dest, help=help, - metavar=metavar) - - def __init__(self, - option_strings, - prog, - parser_class, - dest=SUPPRESS, - help=None, - metavar=None): - - self._prog_prefix = prog - self._parser_class = parser_class - self._name_parser_map = _collections.OrderedDict() - self._choices_actions = [] - - super(_SubParsersAction, self).__init__( - option_strings=option_strings, - dest=dest, - nargs=PARSER, - choices=self._name_parser_map, - help=help, - metavar=metavar) - - def add_parser(self, name, **kwargs): - # set prog from the existing prefix - if kwargs.get('prog') is None: - kwargs['prog'] = '%s %s' % (self._prog_prefix, name) - - aliases = kwargs.pop('aliases', ()) - - # create a pseudo-action to hold the choice help - if 'help' in kwargs: - help = kwargs.pop('help') - choice_action = self._ChoicesPseudoAction(name, aliases, help) - self._choices_actions.append(choice_action) - - # create the parser and add it to the map - parser = self._parser_class(**kwargs) - self._name_parser_map[name] = parser - - # make parser available under aliases also - for alias in aliases: - self._name_parser_map[alias] = parser - - return parser - - def _get_subactions(self): - return self._choices_actions - - def __call__(self, parser, namespace, values, option_string=None): - parser_name = values[0] - arg_strings = values[1:] - - # set the parser name if requested - if self.dest is not SUPPRESS: - setattr(namespace, self.dest, parser_name) - - # select the parser - try: - parser = self._name_parser_map[parser_name] - except KeyError: - args = {'parser_name': parser_name, - 'choices': ', '.join(self._name_parser_map)} - msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args - raise ArgumentError(self, msg) - - # parse all the remaining options into the namespace - # store any unrecognized options on the object, so that the top - # level parser can decide what to do with them - namespace, arg_strings = parser.parse_known_args(arg_strings, namespace) - if arg_strings: - vars(namespace).setdefault(_UNRECOGNIZED_ARGS_ATTR, []) - getattr(namespace, _UNRECOGNIZED_ARGS_ATTR).extend(arg_strings) - - -# ============== -# Type classes -# ============== - -class FileType(object): - """Factory for creating file object types - - Instances of FileType are typically passed as type= arguments to the - ArgumentParser add_argument() method. - - Keyword Arguments: - - mode -- A string indicating how the file is to be opened. Accepts the - same values as the builtin open() function. - - bufsize -- The file's desired buffer size. Accepts the same values as - the builtin open() function. - """ - - def __init__(self, mode='r', bufsize=-1): - self._mode = mode - self._bufsize = bufsize - - def __call__(self, string): - # the special argument "-" means sys.std{in,out} - if string == '-': - if 'r' in self._mode: - return _sys.stdin - elif 'w' in self._mode: - return _sys.stdout - else: - msg = _('argument "-" with mode %r') % self._mode - raise ValueError(msg) - - # all other arguments are used as file names - try: - return open(string, self._mode, self._bufsize) - except IOError as e: - message = _("can't open '%s': %s") - raise ArgumentTypeError(message % (string, e)) - - def __repr__(self): - args = self._mode, self._bufsize - args_str = ', '.join(repr(arg) for arg in args if arg != -1) - return '%s(%s)' % (type(self).__name__, args_str) - -# =========================== -# Optional and Positional Parsing -# =========================== - -class Namespace(_AttributeHolder): - """Simple object for storing attributes. - - Implements equality by attribute names and values, and provides a simple - string representation. - """ - - def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) - - def __eq__(self, other): - return vars(self) == vars(other) - - def __ne__(self, other): - return not (self == other) - - def __contains__(self, key): - return key in self.__dict__ - - -class _ActionsContainer(object): - - def __init__(self, - description, - prefix_chars, - argument_default, - conflict_handler): - super(_ActionsContainer, self).__init__() - - self.description = description - self.argument_default = argument_default - self.prefix_chars = prefix_chars - self.conflict_handler = conflict_handler - - # set up registries - self._registries = {} - - # register actions - self.register('action', None, _StoreAction) - self.register('action', 'store', _StoreAction) - self.register('action', 'store_const', _StoreConstAction) - self.register('action', 'store_true', _StoreTrueAction) - self.register('action', 'store_false', _StoreFalseAction) - self.register('action', 'append', _AppendAction) - self.register('action', 'append_const', _AppendConstAction) - self.register('action', 'count', _CountAction) - self.register('action', 'help', _HelpAction) - self.register('action', 'version', _VersionAction) - self.register('action', 'parsers', _SubParsersAction) - - # raise an exception if the conflict handler is invalid - self._get_handler() - - # action storage - self._actions = [] - self._option_string_actions = {} - - # groups - self._action_groups = [] - self._mutually_exclusive_groups = [] - - # defaults storage - self._defaults = {} - - # determines whether an "option" looks like a negative number - self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$') - - # whether or not there are any optionals that look like negative - # numbers -- uses a list so it can be shared and edited - self._has_negative_number_optionals = [] - - # ==================== - # Registration methods - # ==================== - def register(self, registry_name, value, object): - registry = self._registries.setdefault(registry_name, {}) - registry[value] = object - - def _registry_get(self, registry_name, value, default=None): - return self._registries[registry_name].get(value, default) - - # ================================== - # Namespace default accessor methods - # ================================== - def set_defaults(self, **kwargs): - self._defaults.update(kwargs) - - # if these defaults match any existing arguments, replace - # the previous default on the object with the new one - for action in self._actions: - if action.dest in kwargs: - action.default = kwargs[action.dest] - - def get_default(self, dest): - for action in self._actions: - if action.dest == dest and action.default is not None: - return action.default - return self._defaults.get(dest, None) - - - # ======================= - # Adding argument actions - # ======================= - def add_argument(self, *args, **kwargs): - """ - add_argument(dest, ..., name=value, ...) - add_argument(option_string, option_string, ..., name=value, ...) - """ - - # if no positional args are supplied or only one is supplied and - # it doesn't look like an option string, parse a positional - # argument - chars = self.prefix_chars - if not args or len(args) == 1 and args[0][0] not in chars: - if args and 'dest' in kwargs: - raise ValueError('dest supplied twice for positional argument') - kwargs = self._get_positional_kwargs(*args, **kwargs) - - # otherwise, we're adding an optional argument - else: - kwargs = self._get_optional_kwargs(*args, **kwargs) - - # if no default was supplied, use the parser-level default - if 'default' not in kwargs: - dest = kwargs['dest'] - if dest in self._defaults: - kwargs['default'] = self._defaults[dest] - elif self.argument_default is not None: - kwargs['default'] = self.argument_default - - # create the action object, and add it to the parser - action_class = self._pop_action_class(kwargs) - if not callable(action_class): - raise ValueError('unknown action "%s"' % (action_class,)) - action = action_class(**kwargs) - - # raise an error if the action type is not callable - type_func = self._registry_get('type', action.type, action.type) - if not callable(type_func): - raise ValueError('%r is not callable' % (type_func,)) - - # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - try: - self._get_formatter()._format_args(action, None) - except TypeError: - raise ValueError("length of metavar tuple does not match nargs") - - return self._add_action(action) - - def add_argument_group(self, *args, **kwargs): - group = _ArgumentGroup(self, *args, **kwargs) - self._action_groups.append(group) - return group - - def add_mutually_exclusive_group(self, **kwargs): - group = _MutuallyExclusiveGroup(self, **kwargs) - self._mutually_exclusive_groups.append(group) - return group - - def _add_action(self, action): - # resolve any conflicts - self._check_conflict(action) - - # add to actions list - self._actions.append(action) - action.container = self - - # index the action by any option strings it has - for option_string in action.option_strings: - self._option_string_actions[option_string] = action - - # set the flag if any option strings look like negative numbers - for option_string in action.option_strings: - if self._negative_number_matcher.match(option_string): - if not self._has_negative_number_optionals: - self._has_negative_number_optionals.append(True) - - # return the created action - return action - - def _remove_action(self, action): - self._actions.remove(action) - - def _add_container_actions(self, container): - # collect groups by titles - title_group_map = {} - for group in self._action_groups: - if group.title in title_group_map: - msg = _('cannot merge actions - two groups are named %r') - raise ValueError(msg % (group.title)) - title_group_map[group.title] = group - - # map each action to its group - group_map = {} - for group in container._action_groups: - - # if a group with the title exists, use that, otherwise - # create a new group matching the container's group - if group.title not in title_group_map: - title_group_map[group.title] = self.add_argument_group( - title=group.title, - description=group.description, - conflict_handler=group.conflict_handler) - - # map the actions to their new group - for action in group._group_actions: - group_map[action] = title_group_map[group.title] - - # add container's mutually exclusive groups - # NOTE: if add_mutually_exclusive_group ever gains title= and - # description= then this code will need to be expanded as above - for group in container._mutually_exclusive_groups: - mutex_group = self.add_mutually_exclusive_group( - required=group.required) - - # map the actions to their new mutex group - for action in group._group_actions: - group_map[action] = mutex_group - - # add all actions to this container or their group - for action in container._actions: - group_map.get(action, self)._add_action(action) - - def _get_positional_kwargs(self, dest, **kwargs): - # make sure required is not specified - if 'required' in kwargs: - msg = _("'required' is an invalid argument for positionals") - raise TypeError(msg) - - # mark positional arguments as required if at least one is - # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: - kwargs['required'] = True - - # return the keyword arguments with no option strings - return dict(kwargs, dest=dest, option_strings=[]) - - def _get_optional_kwargs(self, *args, **kwargs): - # determine short and long option strings - option_strings = [] - long_option_strings = [] - for option_string in args: - # error on strings that don't start with an appropriate prefix - if not option_string[0] in self.prefix_chars: - args = {'option': option_string, - 'prefix_chars': self.prefix_chars} - msg = _('invalid option string %(option)r: ' - 'must start with a character %(prefix_chars)r') - raise ValueError(msg % args) - - # strings starting with two prefix characters are long options - option_strings.append(option_string) - if option_string[0] in self.prefix_chars: - if len(option_string) > 1: - if option_string[1] in self.prefix_chars: - long_option_strings.append(option_string) - - # infer destination, '--foo-bar' -> 'foo_bar' and '-x' -> 'x' - dest = kwargs.pop('dest', None) - if dest is None: - if long_option_strings: - dest_option_string = long_option_strings[0] - else: - dest_option_string = option_strings[0] - dest = dest_option_string.lstrip(self.prefix_chars) - if not dest: - msg = _('dest= is required for options like %r') - raise ValueError(msg % option_string) - dest = dest.replace('-', '_') - - # return the updated keyword arguments - return dict(kwargs, dest=dest, option_strings=option_strings) - - def _pop_action_class(self, kwargs, default=None): - action = kwargs.pop('action', default) - return self._registry_get('action', action, action) - - def _get_handler(self): - # determine function from conflict handler string - handler_func_name = '_handle_conflict_%s' % self.conflict_handler - try: - return getattr(self, handler_func_name) - except AttributeError: - msg = _('invalid conflict_resolution value: %r') - raise ValueError(msg % self.conflict_handler) - - def _check_conflict(self, action): - - # find all options that conflict with this option - confl_optionals = [] - for option_string in action.option_strings: - if option_string in self._option_string_actions: - confl_optional = self._option_string_actions[option_string] - confl_optionals.append((option_string, confl_optional)) - - # resolve any conflicts - if confl_optionals: - conflict_handler = self._get_handler() - conflict_handler(action, confl_optionals) - - def _handle_conflict_error(self, action, conflicting_actions): - message = ngettext('conflicting option string: %s', - 'conflicting option strings: %s', - len(conflicting_actions)) - conflict_string = ', '.join([option_string - for option_string, action - in conflicting_actions]) - raise ArgumentError(action, message % conflict_string) - - def _handle_conflict_resolve(self, action, conflicting_actions): - - # remove all conflicting options - for option_string, action in conflicting_actions: - - # remove the conflicting option - action.option_strings.remove(option_string) - self._option_string_actions.pop(option_string, None) - - # if the option now has no option string, remove it from the - # container holding it - if not action.option_strings: - action.container._remove_action(action) - - -class _ArgumentGroup(_ActionsContainer): - - def __init__(self, container, title=None, description=None, **kwargs): - # add any missing keyword arguments by checking the container - update = kwargs.setdefault - update('conflict_handler', container.conflict_handler) - update('prefix_chars', container.prefix_chars) - update('argument_default', container.argument_default) - super_init = super(_ArgumentGroup, self).__init__ - super_init(description=description, **kwargs) - - # group attributes - self.title = title - self._group_actions = [] - - # share most attributes with the container - self._registries = container._registries - self._actions = container._actions - self._option_string_actions = container._option_string_actions - self._defaults = container._defaults - self._has_negative_number_optionals = \ - container._has_negative_number_optionals - self._mutually_exclusive_groups = container._mutually_exclusive_groups - - def _add_action(self, action): - action = super(_ArgumentGroup, self)._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - super(_ArgumentGroup, self)._remove_action(action) - self._group_actions.remove(action) - - -class _MutuallyExclusiveGroup(_ArgumentGroup): - - def __init__(self, container, required=False): - super(_MutuallyExclusiveGroup, self).__init__(container) - self.required = required - self._container = container - - def _add_action(self, action): - if action.required: - msg = _('mutually exclusive arguments must be optional') - raise ValueError(msg) - action = self._container._add_action(action) - self._group_actions.append(action) - return action - - def _remove_action(self, action): - self._container._remove_action(action) - self._group_actions.remove(action) - - -class ArgumentParser(_AttributeHolder, _ActionsContainer): - """Object for parsing command line strings into Python objects. - - Keyword Arguments: - - prog -- The name of the program (default: sys.argv[0]) - - usage -- A usage message (default: auto-generated from arguments) - - description -- A description of what the program does - - epilog -- Text following the argument descriptions - - parents -- Parsers whose arguments should be copied into this one - - formatter_class -- HelpFormatter class for printing help messages - - prefix_chars -- Characters that prefix optional arguments - - fromfile_prefix_chars -- Characters that prefix files containing - additional arguments - - argument_default -- The default value for all arguments - - conflict_handler -- String indicating how to handle conflicts - - add_help -- Add a -h/-help option - """ - - def __init__(self, - prog=None, - usage=None, - description=None, - epilog=None, - version=None, - parents=[], - formatter_class=HelpFormatter, - prefix_chars='-', - fromfile_prefix_chars=None, - argument_default=None, - conflict_handler='error', - add_help=True): - - if version is not None: - import warnings - warnings.warn( - """The "version" argument to ArgumentParser is deprecated. """ - """Please use """ - """"add_argument(..., action='version', version="N", ...)" """ - """instead""", DeprecationWarning) - - superinit = super(ArgumentParser, self).__init__ - superinit(description=description, - prefix_chars=prefix_chars, - argument_default=argument_default, - conflict_handler=conflict_handler) - - # default setting for prog - if prog is None: - prog = _os.path.basename(_sys.argv[0]) - - self.prog = prog - self.usage = usage - self.epilog = epilog - self.version = version - self.formatter_class = formatter_class - self.fromfile_prefix_chars = fromfile_prefix_chars - self.add_help = add_help - - add_group = self.add_argument_group - self._positionals = add_group(_('positional arguments')) - self._optionals = add_group(_('optional arguments')) - self._subparsers = None - - # register types - def identity(string): - return string - self.register('type', None, identity) - - # add help and version arguments if necessary - # (using explicit default to override global argument_default) - default_prefix = '-' if '-' in prefix_chars else prefix_chars[0] - if self.add_help: - self.add_argument( - default_prefix+'h', default_prefix*2+'help', - action='help', default=SUPPRESS, - help=_('show this help message and exit')) - if self.version: - self.add_argument( - default_prefix+'v', default_prefix*2+'version', - action='version', default=SUPPRESS, - version=self.version, - help=_("show program's version number and exit")) - - # add parent arguments and defaults - for parent in parents: - self._add_container_actions(parent) - try: - defaults = parent._defaults - except AttributeError: - pass - else: - self._defaults.update(defaults) - - # ======================= - # Pretty __repr__ methods - # ======================= - def _get_kwargs(self): - names = [ - 'prog', - 'usage', - 'description', - 'version', - 'formatter_class', - 'conflict_handler', - 'add_help', - ] - return [(name, getattr(self, name)) for name in names] - - # ================================== - # Optional/Positional adding methods - # ================================== - def add_subparsers(self, **kwargs): - if self._subparsers is not None: - self.error(_('cannot have multiple subparser arguments')) - - # add the parser class to the arguments if it's not present - kwargs.setdefault('parser_class', type(self)) - - if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) - self._subparsers = self.add_argument_group(title, description) - else: - self._subparsers = self._positionals - - # prog defaults to the usage message of this parser, skipping - # optional arguments and with no "usage:" prefix - if kwargs.get('prog') is None: - formatter = self._get_formatter() - positionals = self._get_positional_actions() - groups = self._mutually_exclusive_groups - formatter.add_usage(self.usage, positionals, groups, '') - kwargs['prog'] = formatter.format_help().strip() - - # create the parsers action and add it to the positionals list - parsers_class = self._pop_action_class(kwargs, 'parsers') - action = parsers_class(option_strings=[], **kwargs) - self._subparsers._add_action(action) - - # return the created parsers action - return action - - def _add_action(self, action): - if action.option_strings: - self._optionals._add_action(action) - else: - self._positionals._add_action(action) - return action - - def _get_optional_actions(self): - return [action - for action in self._actions - if action.option_strings] - - def _get_positional_actions(self): - return [action - for action in self._actions - if not action.option_strings] - - # ===================================== - # Command line argument parsing methods - # ===================================== - def parse_args(self, args=None, namespace=None): - args, argv = self.parse_known_args(args, namespace) - if argv: - msg = _('unrecognized arguments: %s') - self.error(msg % ' '.join(argv)) - return args - - def parse_known_args(self, args=None, namespace=None): - if args is None: - # args default to the system args - args = _sys.argv[1:] - else: - # make sure that args are mutable - args = list(args) - - # default Namespace built from parser defaults - if namespace is None: - namespace = Namespace() - - # add any action defaults that aren't present - for action in self._actions: - if action.dest is not SUPPRESS: - if not hasattr(namespace, action.dest): - if action.default is not SUPPRESS: - setattr(namespace, action.dest, action.default) - - # add any parser defaults that aren't present - for dest in self._defaults: - if not hasattr(namespace, dest): - setattr(namespace, dest, self._defaults[dest]) - - # parse the arguments and exit if there are any errors - try: - namespace, args = self._parse_known_args(args, namespace) - if hasattr(namespace, _UNRECOGNIZED_ARGS_ATTR): - args.extend(getattr(namespace, _UNRECOGNIZED_ARGS_ATTR)) - delattr(namespace, _UNRECOGNIZED_ARGS_ATTR) - return namespace, args - except ArgumentError: - err = _sys.exc_info()[1] - self.error(str(err)) - - def _parse_known_args(self, arg_strings, namespace): - # replace arg strings that are file references - if self.fromfile_prefix_chars is not None: - arg_strings = self._read_args_from_files(arg_strings) - - # map all mutually exclusive arguments to the other arguments - # they can't occur with - action_conflicts = {} - for mutex_group in self._mutually_exclusive_groups: - group_actions = mutex_group._group_actions - for i, mutex_action in enumerate(mutex_group._group_actions): - conflicts = action_conflicts.setdefault(mutex_action, []) - conflicts.extend(group_actions[:i]) - conflicts.extend(group_actions[i + 1:]) - - # find all option indices, and determine the arg_string_pattern - # which has an 'O' if there is an option at an index, - # an 'A' if there is an argument, or a '-' if there is a '--' - option_string_indices = {} - arg_string_pattern_parts = [] - arg_strings_iter = iter(arg_strings) - for i, arg_string in enumerate(arg_strings_iter): - - # all args after -- are non-options - if arg_string == '--': - arg_string_pattern_parts.append('-') - for arg_string in arg_strings_iter: - arg_string_pattern_parts.append('A') - - # otherwise, add the arg to the arg strings - # and note the index if it was an option - else: - option_tuple = self._parse_optional(arg_string) - if option_tuple is None: - pattern = 'A' - else: - option_string_indices[i] = option_tuple - pattern = 'O' - arg_string_pattern_parts.append(pattern) - - # join the pieces together to form the pattern - arg_strings_pattern = ''.join(arg_string_pattern_parts) - - # converts arg strings to the appropriate and then takes the action - seen_actions = set() - seen_non_default_actions = set() - - def take_action(action, argument_strings, option_string=None): - seen_actions.add(action) - argument_values = self._get_values(action, argument_strings) - - # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: - seen_non_default_actions.add(action) - for conflict_action in action_conflicts.get(action, []): - if conflict_action in seen_non_default_actions: - msg = _('not allowed with argument %s') - action_name = _get_action_name(conflict_action) - raise ArgumentError(action, msg % action_name) - - # take the action if we didn't receive a SUPPRESS value - # (e.g. from a default) - if argument_values is not SUPPRESS: - action(self, namespace, argument_values, option_string) - - # function to convert arg_strings into an optional action - def consume_optional(start_index): - - # get the optional identified at this index - option_tuple = option_string_indices[start_index] - action, option_string, explicit_arg = option_tuple - - # identify additional optionals in the same arg string - # (e.g. -xyz is the same as -x -y -z if no args are required) - match_argument = self._match_argument - action_tuples = [] - while True: - - # if we found no optional action, skip it - if action is None: - extras.append(arg_strings[start_index]) - return start_index + 1 - - # if there is an explicit argument, try to match the - # optional's string arguments to only this - if explicit_arg is not None: - arg_count = match_argument(action, 'A') - - # if the action is a single-dash option and takes no - # arguments, try to parse more single-dash options out - # of the tail of the option string - chars = self.prefix_chars - if arg_count == 0 and option_string[1] not in chars: - action_tuples.append((action, [], option_string)) - char = option_string[0] - option_string = char + explicit_arg[0] - new_explicit_arg = explicit_arg[1:] or None - optionals_map = self._option_string_actions - if option_string in optionals_map: - action = optionals_map[option_string] - explicit_arg = new_explicit_arg - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if the action expect exactly one argument, we've - # successfully matched the option; exit the loop - elif arg_count == 1: - stop = start_index + 1 - args = [explicit_arg] - action_tuples.append((action, args, option_string)) - break - - # error if a double-dash option did not use the - # explicit argument - else: - msg = _('ignored explicit argument %r') - raise ArgumentError(action, msg % explicit_arg) - - # if there is no explicit argument, try to match the - # optional's string arguments with the following strings - # if successful, exit the loop - else: - start = start_index + 1 - selected_patterns = arg_strings_pattern[start:] - arg_count = match_argument(action, selected_patterns) - stop = start + arg_count - args = arg_strings[start:stop] - action_tuples.append((action, args, option_string)) - break - - # add the Optional to the list and return the index at which - # the Optional's string args stopped - assert action_tuples - for action, args, option_string in action_tuples: - take_action(action, args, option_string) - return stop - - # the list of Positionals left to be parsed; this is modified - # by consume_positionals() - positionals = self._get_positional_actions() - - # function to convert arg_strings into positional actions - def consume_positionals(start_index): - # match as many Positionals as possible - match_partial = self._match_arguments_partial - selected_pattern = arg_strings_pattern[start_index:] - arg_counts = match_partial(positionals, selected_pattern) - - # slice off the appropriate arg strings for each Positional - # and add the Positional and its args to the list - for action, arg_count in zip(positionals, arg_counts): - args = arg_strings[start_index: start_index + arg_count] - start_index += arg_count - take_action(action, args) - - # slice off the Positionals that we just parsed and return the - # index at which the Positionals' string args stopped - positionals[:] = positionals[len(arg_counts):] - return start_index - - # consume Positionals and Optionals alternately, until we have - # passed the last option string - extras = [] - start_index = 0 - if option_string_indices: - max_option_string_index = max(option_string_indices) - else: - max_option_string_index = -1 - while start_index <= max_option_string_index: - - # consume any Positionals preceding the next option - next_option_string_index = min([ - index - for index in option_string_indices - if index >= start_index]) - if start_index != next_option_string_index: - positionals_end_index = consume_positionals(start_index) - - # only try to parse the next optional if we didn't consume - # the option string during the positionals parsing - if positionals_end_index > start_index: - start_index = positionals_end_index - continue - else: - start_index = positionals_end_index - - # if we consumed all the positionals we could and we're not - # at the index of an option string, there were extra arguments - if start_index not in option_string_indices: - strings = arg_strings[start_index:next_option_string_index] - extras.extend(strings) - start_index = next_option_string_index - - # consume the next optional and any arguments for it - start_index = consume_optional(start_index) - - # consume any positionals following the last Optional - stop_index = consume_positionals(start_index) - - # if we didn't consume all the argument strings, there were extras - extras.extend(arg_strings[stop_index:]) - - # if we didn't use all the Positional objects, there were too few - # arg strings supplied. - if positionals: - self.error(_('too few arguments')) - - # make sure all required actions were present, and convert defaults. - for action in self._actions: - if action not in seen_actions: - if action.required: - name = _get_action_name(action) - self.error(_('argument %s is required') % name) - else: - # Convert action default now instead of doing it before - # parsing arguments to avoid calling convert functions - # twice (which may fail) if the argument was given, but - # only if it was defined already in the namespace - if (action.default is not None and - isinstance(action.default, str) and - hasattr(namespace, action.dest) and - action.default is getattr(namespace, action.dest)): - setattr(namespace, action.dest, - self._get_value(action, action.default)) - - # make sure all required groups had one option present - for group in self._mutually_exclusive_groups: - if group.required: - for action in group._group_actions: - if action in seen_non_default_actions: - break - - # if no actions were used, report the error - else: - names = [_get_action_name(action) - for action in group._group_actions - if action.help is not SUPPRESS] - msg = _('one of the arguments %s is required') - self.error(msg % ' '.join(names)) - - # return the updated namespace and the extra arguments - return namespace, extras - - def _read_args_from_files(self, arg_strings): - # expand arguments referencing files - new_arg_strings = [] - for arg_string in arg_strings: - - # for regular arguments, just add them back into the list - if not arg_string or arg_string[0] not in self.fromfile_prefix_chars: - new_arg_strings.append(arg_string) - - # replace arguments referencing files with the file content - else: - try: - args_file = open(arg_string[1:]) - try: - arg_strings = [] - for arg_line in args_file.read().splitlines(): - for arg in self.convert_arg_line_to_args(arg_line): - arg_strings.append(arg) - arg_strings = self._read_args_from_files(arg_strings) - new_arg_strings.extend(arg_strings) - finally: - args_file.close() - except IOError: - err = _sys.exc_info()[1] - self.error(str(err)) - - # return the modified argument list - return new_arg_strings - - def convert_arg_line_to_args(self, arg_line): - return [arg_line] - - def _match_argument(self, action, arg_strings_pattern): - # match the pattern for this action to the arg strings - nargs_pattern = self._get_nargs_pattern(action) - match = _re.match(nargs_pattern, arg_strings_pattern) - - # raise an exception if we weren't able to find a match - if match is None: - nargs_errors = { - None: _('expected one argument'), - OPTIONAL: _('expected at most one argument'), - ONE_OR_MORE: _('expected at least one argument'), - } - default = ngettext('expected %s argument', - 'expected %s arguments', - action.nargs) % action.nargs - msg = nargs_errors.get(action.nargs, default) - raise ArgumentError(action, msg) - - # return the number of arguments matched - return len(match.group(1)) - - def _match_arguments_partial(self, actions, arg_strings_pattern): - # progressively shorten the actions list by slicing off the - # final actions until we find a match - result = [] - for i in range(len(actions), 0, -1): - actions_slice = actions[:i] - pattern = ''.join([self._get_nargs_pattern(action) - for action in actions_slice]) - match = _re.match(pattern, arg_strings_pattern) - if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result - - def _parse_optional(self, arg_string): - # if it's an empty string, it was meant to be a positional - if not arg_string: - return None - - # if it doesn't start with a prefix, it was meant to be positional - if not arg_string[0] in self.prefix_chars: - return None - - # if the option string is present in the parser, return the action - if arg_string in self._option_string_actions: - action = self._option_string_actions[arg_string] - return action, arg_string, None - - # if it's just a single character, it was meant to be positional - if len(arg_string) == 1: - return None - - # if the option string before the "=" is present, return the action - if '=' in arg_string: - option_string, explicit_arg = arg_string.split('=', 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg - - # search through all possible prefixes of the option string - # and all actions in the parser for possible interpretations - option_tuples = self._get_option_tuples(arg_string) - - # if multiple actions match, the option string was ambiguous - if len(option_tuples) > 1: - options = ', '.join([option_string - for action, option_string, explicit_arg in option_tuples]) - args = {'option': arg_string, 'matches': options} - msg = _('ambiguous option: %(option)s could match %(matches)s') - self.error(msg % args) - - # if exactly one action matched, this segmentation is good, - # so return the parsed action - elif len(option_tuples) == 1: - option_tuple, = option_tuples - return option_tuple - - # if it was not found as an option, but it looks like a negative - # number, it was meant to be positional - # unless there are negative-number-like options - if self._negative_number_matcher.match(arg_string): - if not self._has_negative_number_optionals: - return None - - # if it contains a space, it was meant to be a positional - if ' ' in arg_string: - return None - - # it was meant to be an optional but there is no such option - # in this parser (though it might be a valid option in a subparser) - return None, arg_string, None - - def _get_option_tuples(self, option_string): - result = [] - - # option strings starting with two prefix characters are only - # split at the '=' - chars = self.prefix_chars - if option_string[0] in chars and option_string[1] in chars: - if '=' in option_string: - option_prefix, explicit_arg = option_string.split('=', 1) - else: - option_prefix = option_string - explicit_arg = None - for option_string in self._option_string_actions: - if option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # single character options can be concatenated with their arguments - # but multiple character options always have to have their argument - # separate - elif option_string[0] in chars and option_string[1] not in chars: - option_prefix = option_string - explicit_arg = None - short_option_prefix = option_string[:2] - short_explicit_arg = option_string[2:] - - for option_string in self._option_string_actions: - if option_string == short_option_prefix: - action = self._option_string_actions[option_string] - tup = action, option_string, short_explicit_arg - result.append(tup) - elif option_string.startswith(option_prefix): - action = self._option_string_actions[option_string] - tup = action, option_string, explicit_arg - result.append(tup) - - # shouldn't ever get here - else: - self.error(_('unexpected option string: %s') % option_string) - - # return the collected option tuples - return result - - def _get_nargs_pattern(self, action): - # in all examples below, we have to allow for '--' args - # which are represented as '-' in the pattern - nargs = action.nargs - - # the default (None) is assumed to be a single argument - if nargs is None: - nargs_pattern = '(-*A-*)' - - # allow zero or one arguments - elif nargs == OPTIONAL: - nargs_pattern = '(-*A?-*)' - - # allow zero or more arguments - elif nargs == ZERO_OR_MORE: - nargs_pattern = '(-*[A-]*)' - - # allow one or more arguments - elif nargs == ONE_OR_MORE: - nargs_pattern = '(-*A[A-]*)' - - # allow any number of options or arguments - elif nargs == REMAINDER: - nargs_pattern = '([-AO]*)' - - # allow one argument followed by any number of options or arguments - elif nargs == PARSER: - nargs_pattern = '(-*A[-AO]*)' - - # all others should be integers - else: - nargs_pattern = '(-*%s-*)' % '-*'.join('A' * nargs) - - # if this is an optional action, -- is not allowed - if action.option_strings: - nargs_pattern = nargs_pattern.replace('-*', '') - nargs_pattern = nargs_pattern.replace('-', '') - - # return the pattern - return nargs_pattern - - # ======================== - # Value conversion methods - # ======================== - def _get_values(self, action, arg_strings): - # for everything but PARSER, REMAINDER args, strip out first '--' - if action.nargs not in [PARSER, REMAINDER]: - try: - arg_strings.remove('--') - except ValueError: - pass - - # optional argument produces a default when not present - if not arg_strings and action.nargs == OPTIONAL: - if action.option_strings: - value = action.const - else: - value = action.default - if isinstance(value, str): - value = self._get_value(action, value) - self._check_value(action, value) - - # when nargs='*' on a positional, if there were no command-line - # args, use the default if it is anything other than None - elif (not arg_strings and action.nargs == ZERO_OR_MORE and - not action.option_strings): - if action.default is not None: - value = action.default - else: - value = arg_strings - self._check_value(action, value) - - # single argument or optional argument produces a single value - elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]: - arg_string, = arg_strings - value = self._get_value(action, arg_string) - self._check_value(action, value) - - # REMAINDER arguments convert all values, checking none - elif action.nargs == REMAINDER: - value = [self._get_value(action, v) for v in arg_strings] - - # PARSER arguments convert all values, but check only the first - elif action.nargs == PARSER: - value = [self._get_value(action, v) for v in arg_strings] - self._check_value(action, value[0]) - - # all other types of nargs produce a list - else: - value = [self._get_value(action, v) for v in arg_strings] - for v in value: - self._check_value(action, v) - - # return the converted value - return value - - def _get_value(self, action, arg_string): - type_func = self._registry_get('type', action.type, action.type) - if not callable(type_func): - msg = _('%r is not callable') - raise ArgumentError(action, msg % type_func) - - # convert the value to the appropriate type - try: - result = type_func(arg_string) - - # ArgumentTypeErrors indicate errors - except ArgumentTypeError: - name = getattr(action.type, '__name__', repr(action.type)) - msg = str(_sys.exc_info()[1]) - raise ArgumentError(action, msg) - - # TypeErrors or ValueErrors also indicate errors - except (TypeError, ValueError): - name = getattr(action.type, '__name__', repr(action.type)) - args = {'type': name, 'value': arg_string} - msg = _('invalid %(type)s value: %(value)r') - raise ArgumentError(action, msg % args) - - # return the converted value - return result - - def _check_value(self, action, value): - # converted value must be one of the choices (if specified) - if action.choices is not None and value not in action.choices: - args = {'value': value, - 'choices': ', '.join(map(repr, action.choices))} - msg = _('invalid choice: %(value)r (choose from %(choices)s)') - raise ArgumentError(action, msg % args) - - # ======================= - # Help-formatting methods - # ======================= - def format_usage(self): - formatter = self._get_formatter() - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - return formatter.format_help() - - def format_help(self): - formatter = self._get_formatter() - - # usage - formatter.add_usage(self.usage, self._actions, - self._mutually_exclusive_groups) - - # description - formatter.add_text(self.description) - - # positionals, optionals and user-defined groups - for action_group in self._action_groups: - formatter.start_section(action_group.title) - formatter.add_text(action_group.description) - formatter.add_arguments(action_group._group_actions) - formatter.end_section() - - # epilog - formatter.add_text(self.epilog) - - # determine help from format above - return formatter.format_help() - - def format_version(self): - import warnings - warnings.warn( - 'The format_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - formatter = self._get_formatter() - formatter.add_text(self.version) - return formatter.format_help() - - def _get_formatter(self): - return self.formatter_class(prog=self.prog) - - # ===================== - # Help-printing methods - # ===================== - def print_usage(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_usage(), file) - - def print_help(self, file=None): - if file is None: - file = _sys.stdout - self._print_message(self.format_help(), file) - - def print_version(self, file=None): - import warnings - warnings.warn( - 'The print_version method is deprecated -- the "version" ' - 'argument to ArgumentParser is no longer supported.', - DeprecationWarning) - self._print_message(self.format_version(), file) - - def _print_message(self, message, file=None): - if message: - if file is None: - file = _sys.stderr - file.write(message) - - # =============== - # Exiting methods - # =============== - def exit(self, status=0, message=None): - if message: - self._print_message(message, _sys.stderr) - _sys.exit(status) - - def error(self, message): - """error(message: string) - - Prints a usage message incorporating the message to stderr and - exits. - - If you override this in a subclass, it should not return -- it - should either exit or raise an exception. - """ - self.print_usage(_sys.stderr) - args = {'prog': self.prog, 'message': message} - self.exit(2, _('%(prog)s: error: %(message)s\n') % args) diff --git a/setup.py b/setup.py index 6a4d34bd..b7a43daa 100644 --- a/setup.py +++ b/setup.py @@ -247,7 +247,7 @@ setup( 'Topic :: Security :: Cryptography', 'Topic :: System :: Archiving :: Backup', ], - packages=['borg', 'borg.testsuite', 'borg.support', ], + packages=['borg', 'borg.testsuite', ], entry_points={ 'console_scripts': [ 'borg = borg.archiver:main', From dabac6a4ed3a8b0794e66fe41acdd93b3a101618 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 22:08:53 +0100 Subject: [PATCH 270/321] use mock from stdlib, fixes #145 --- borg/testsuite/archive.py | 2 +- borg/testsuite/archiver.py | 2 +- borg/testsuite/repository.py | 3 +-- requirements.d/development.txt | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/borg/testsuite/archive.py b/borg/testsuite/archive.py index a963573e..662d776c 100644 --- a/borg/testsuite/archive.py +++ b/borg/testsuite/archive.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone +from unittest.mock import Mock import msgpack -from mock import Mock from ..archive import Archive, CacheChunkBuffer, RobustUnpacker from ..key import PlaintextKey diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index f75cc120..74e41152 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -11,9 +11,9 @@ import shutil import tempfile import time import unittest +from unittest.mock import patch from hashlib import sha256 -from mock import patch import pytest from .. import xattr diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 7027cb59..3da7b80f 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -2,8 +2,7 @@ import os import shutil import sys import tempfile - -from mock import patch +from unittest.mock import patch from ..hashindex import NSIndex from ..helpers import Location, IntegrityError diff --git a/requirements.d/development.txt b/requirements.d/development.txt index bf21d52a..7c1624b4 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -1,6 +1,5 @@ virtualenv<14.0 tox -mock pytest pytest-cov<2.0.0 pytest-benchmark>=3.0.0 From 19998888ba14d48850ec3a7665da578c8f3f7451 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 22:15:32 +0100 Subject: [PATCH 271/321] remove support for missing PermissionError on py 3.2 --- borg/archive.py | 6 ------ borg/testsuite/archiver.py | 6 ------ 2 files changed, 12 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index f5a3d296..e8fe2fed 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -43,12 +43,6 @@ has_mtime_ns = sys.version >= '3.3' has_lchmod = hasattr(os, 'lchmod') has_lchflags = hasattr(os, 'lchflags') -# Python <= 3.2 raises OSError instead of PermissionError (See #164) -try: - PermissionError = PermissionError -except NameError: - PermissionError = OSError - class DownloadPipeline: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 74e41152..2b27a516 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -36,12 +36,6 @@ has_lchflags = hasattr(os, 'lchflags') src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..') -# Python <= 3.2 raises OSError instead of PermissionError (See #164) -try: - PermissionError = PermissionError -except NameError: - PermissionError = OSError - def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): if fork: From 265da6286f0045843ac0e805254c262449b9ee8b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 22:39:43 +0100 Subject: [PATCH 272/321] remove conditionals/wrappers, we always have stat nanosecond support on 3.4+ also: no wrapper needed for binascii.unhexlify any more --- borg/archive.py | 9 ++++----- borg/archiver.py | 4 ++-- borg/cache.py | 8 ++++---- borg/helpers.py | 29 ----------------------------- borg/repository.py | 4 ++-- borg/testsuite/__init__.py | 10 ++++------ borg/testsuite/archiver.py | 8 ++++---- borg/testsuite/key.py | 4 ++-- 8 files changed, 22 insertions(+), 54 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index e8fe2fed..f6acf8be 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -18,7 +18,7 @@ from io import BytesIO from . import xattr from .helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, format_timedelta, \ Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ - st_atime_ns, st_ctime_ns, st_mtime_ns, ProgressIndicatorPercent + ProgressIndicatorPercent from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex @@ -39,7 +39,6 @@ ITEMS_CHUNKER_PARAMS = (12, 16, 14, HASH_WINDOW_SIZE) utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {}) -has_mtime_ns = sys.version >= '3.3' has_lchmod = hasattr(os, 'lchmod') has_lchflags = hasattr(os, 'lchflags') @@ -435,9 +434,9 @@ Number of files: {0.stats.nfiles}'''.format(self) b'mode': st.st_mode, b'uid': st.st_uid, b'user': uid2user(st.st_uid), b'gid': st.st_gid, b'group': gid2group(st.st_gid), - b'atime': int_to_bigint(st_atime_ns(st)), - b'ctime': int_to_bigint(st_ctime_ns(st)), - b'mtime': int_to_bigint(st_mtime_ns(st)), + b'atime': int_to_bigint(st.st_atime_ns), + b'ctime': int_to_bigint(st.st_ctime_ns), + b'mtime': int_to_bigint(st.st_mtime_ns), } if self.numeric_owner: item[b'user'] = item[b'group'] = None diff --git a/borg/archiver.py b/borg/archiver.py index ffb0727c..e595715b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1,4 +1,4 @@ -from binascii import hexlify +from binascii import hexlify, unhexlify from datetime import datetime from hashlib import sha256 from operator import attrgetter @@ -16,7 +16,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ - get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ + get_cache_dir, get_keys_dir, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher diff --git a/borg/cache.py b/borg/cache.py index cbefcd5e..feffc1fd 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -3,13 +3,13 @@ from .remote import cache_if_remote from collections import namedtuple import os import stat -from binascii import hexlify +from binascii import hexlify, unhexlify import shutil from .key import PlaintextKey from .logger import create_logger logger = create_logger() -from .helpers import Error, get_cache_dir, decode_dict, st_mtime_ns, unhexlify, int_to_bigint, \ +from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \ bigint_to_int, format_file_size, yes from .locking import UpgradableLock from .hashindex import ChunkIndex @@ -401,7 +401,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not entry: return None entry = msgpack.unpackb(entry) - if entry[2] == st.st_size and bigint_to_int(entry[3]) == st_mtime_ns(st) and entry[1] == st.st_ino: + if entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and entry[1] == st.st_ino: # reset entry age entry[0] = 0 self.files[path_hash] = msgpack.packb(entry) @@ -413,6 +413,6 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not (self.do_files and stat.S_ISREG(st.st_mode)): return # Entry: Age, inode, size, mtime, chunk ids - mtime_ns = st_mtime_ns(st) + mtime_ns = st.st_mtime_ns self.files[path_hash] = msgpack.packb((0, st.st_ino, st.st_size, int_to_bigint(mtime_ns), ids)) self._newest_mtime = max(self._newest_mtime, mtime_ns) diff --git a/borg/helpers.py b/borg/helpers.py index 8670419b..89968c8c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -835,35 +835,6 @@ class StableDict(dict): return sorted(super().items()) -if sys.version < '3.3': - # st_xtime_ns attributes only available in 3.3+ - def st_atime_ns(st): - return int(st.st_atime * 1e9) - - def st_ctime_ns(st): - return int(st.st_ctime * 1e9) - - def st_mtime_ns(st): - return int(st.st_mtime * 1e9) - - # unhexlify in < 3.3 incorrectly only accepts bytes input - def unhexlify(data): - if isinstance(data, str): - data = data.encode('ascii') - return binascii.unhexlify(data) -else: - def st_atime_ns(st): - return st.st_atime_ns - - def st_ctime_ns(st): - return st.st_ctime_ns - - def st_mtime_ns(st): - return st.st_mtime_ns - - unhexlify = binascii.unhexlify - - def bigint_to_int(mtime): """Convert bytearray to int """ diff --git a/borg/repository.py b/borg/repository.py index 80bbe669..0789a021 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -1,5 +1,5 @@ from configparser import ConfigParser -from binascii import hexlify +from binascii import hexlify, unhexlify from itertools import islice import errno import logging @@ -11,7 +11,7 @@ import struct from zlib import crc32 import msgpack -from .helpers import Error, ErrorWithTraceback, IntegrityError, unhexlify, ProgressIndicatorPercent +from .helpers import Error, ErrorWithTraceback, IntegrityError, ProgressIndicatorPercent from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 3af1738b..cb643153 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -7,7 +7,6 @@ import sys import sysconfig import time import unittest -from ..helpers import st_mtime_ns from ..xattr import get_all try: @@ -31,7 +30,6 @@ else: if sys.platform.startswith('netbsd'): st_mtime_ns_round = -4 # only >1 microsecond resolution here? -has_mtime_ns = sys.version >= '3.3' utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) @@ -83,11 +81,11 @@ class BaseTestCase(unittest.TestCase): if not os.path.islink(path1) or utime_supports_fd: # Older versions of llfuse do not support ns precision properly if fuse and not have_fuse_mtime_ns: - d1.append(round(st_mtime_ns(s1), -4)) - d2.append(round(st_mtime_ns(s2), -4)) + d1.append(round(s1.st_mtime_ns, -4)) + d2.append(round(s2.st_mtime_ns, -4)) else: - d1.append(round(st_mtime_ns(s1), st_mtime_ns_round)) - d2.append(round(st_mtime_ns(s2), st_mtime_ns_round)) + d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) + d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) d1.append(get_all(path1, follow_symlinks=False)) d2.append(get_all(path2, follow_symlinks=False)) self.assert_equal(d1, d2) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 2b27a516..3192d60d 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -21,7 +21,7 @@ from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP from ..archiver import Archiver from ..cache import Cache from ..crypto import bytes_to_long, num_aes_blocks -from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, st_atime_ns, st_mtime_ns +from ..helpers import Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import BaseTestCase, changedir, environment_variable @@ -358,12 +358,12 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', self.repository_location + '::test') sti = os.stat('input/file1') sto = os.stat('output/input/file1') - assert st_mtime_ns(sti) == st_mtime_ns(sto) == mtime * 1e9 + assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 if hasattr(os, 'O_NOATIME'): - assert st_atime_ns(sti) == st_atime_ns(sto) == atime * 1e9 + assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9 else: # it touched the input file's atime while backing it up - assert st_atime_ns(sto) == atime * 1e9 + assert sto.st_atime_ns == atime * 1e9 def _extract_repository_id(self, path): return Repository(self.repository_path).id diff --git a/borg/testsuite/key.py b/borg/testsuite/key.py index b2011d8f..4c57d1f0 100644 --- a/borg/testsuite/key.py +++ b/borg/testsuite/key.py @@ -2,11 +2,11 @@ import os import re import shutil import tempfile -from binascii import hexlify +from binascii import hexlify, unhexlify from ..crypto import bytes_to_long, num_aes_blocks from ..key import PlaintextKey, PassphraseKey, KeyfileKey -from ..helpers import Location, unhexlify +from ..helpers import Location from . import BaseTestCase From fe8762ad284c1104331d260d6e0f07e47aed6f92 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 22:53:31 +0100 Subject: [PATCH 273/321] os.utime on py 3.4+ always supports fd and follow_symlinks --- borg/archive.py | 8 ++------ borg/testsuite/__init__.py | 17 +++++++---------- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index f6acf8be..ca653ee8 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -37,8 +37,6 @@ CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE # chunker params for the items metadata stream, finer granularity ITEMS_CHUNKER_PARAMS = (12, 16, 14, HASH_WINDOW_SIZE) -utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) -utime_supports_follow_symlinks = os.utime in getattr(os, 'supports_follow_symlinks', {}) has_lchmod = hasattr(os, 'lchmod') has_lchflags = hasattr(os, 'lchflags') @@ -385,12 +383,10 @@ Number of files: {0.stats.nfiles}'''.format(self) else: # old archives only had mtime in item metadata atime = mtime - if fd and utime_supports_fd: # Python >= 3.3 + if fd: os.utime(fd, None, ns=(atime, mtime)) - elif utime_supports_follow_symlinks: # Python >= 3.3 + else: os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) - elif not symlink: - os.utime(path, (atime / 1e9, mtime / 1e9)) acl_set(path, item, self.numeric_owner) # Only available on OS X and FreeBSD if has_lchflags and b'bsdflags' in item: diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index cb643153..67c97789 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -30,8 +30,6 @@ else: if sys.platform.startswith('netbsd'): st_mtime_ns_round = -4 # only >1 microsecond resolution here? -utime_supports_fd = os.utime in getattr(os, 'supports_fd', {}) - class BaseTestCase(unittest.TestCase): """ @@ -78,14 +76,13 @@ class BaseTestCase(unittest.TestCase): d1[4] = None if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]): d2[4] = None - if not os.path.islink(path1) or utime_supports_fd: - # Older versions of llfuse do not support ns precision properly - if fuse and not have_fuse_mtime_ns: - d1.append(round(s1.st_mtime_ns, -4)) - d2.append(round(s2.st_mtime_ns, -4)) - else: - d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) - d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) + # Older versions of llfuse do not support ns precision properly + if fuse and not have_fuse_mtime_ns: + d1.append(round(s1.st_mtime_ns, -4)) + d2.append(round(s2.st_mtime_ns, -4)) + else: + d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) + d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) d1.append(get_all(path1, follow_symlinks=False)) d2.append(get_all(path2, follow_symlinks=False)) self.assert_equal(d1, d2) From 6a5629226fc54772a43f8b215232c9eb1421c652 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 22:58:32 +0100 Subject: [PATCH 274/321] simplify to print(...., flush=True) --- borg/helpers.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 89968c8c..0a70fc63 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -210,8 +210,7 @@ class Statistics: msg += "{0:<{space}}".format(path, space=space) else: msg = ' ' * columns - print(msg, file=stream or sys.stderr, end="\r") - (stream or sys.stderr).flush() + print(msg, file=stream or sys.stderr, end="\r", flush=True) def get_keys_dir(): @@ -934,8 +933,7 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, # no retries wanted, we just return the default return default if retry_msg: - print(retry_msg, file=ofile, end='') - ofile.flush() + print(retry_msg, file=ofile, end='', flush=True) class ProgressIndicatorPercent: @@ -973,8 +971,7 @@ class ProgressIndicatorPercent: return self.output(pct) def output(self, percent): - print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n') # python 3.3 gives us flush=True - self.file.flush() + print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n', flush=True) def finish(self): if self.same_line: @@ -1008,8 +1005,7 @@ class ProgressIndicatorEndless: return self.output(self.triggered) def output(self, triggered): - print('.', end='', file=self.file) # python 3.3 gives us flush=True - self.file.flush() + print('.', end='', file=self.file, flush=True) def finish(self): print(file=self.file) From 4444113414e09e8de4e359a70b62e97ecbf6a888 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 23:07:06 +0100 Subject: [PATCH 275/321] remove misc. compat code not needed for py 3.4+ --- borg/compress.pyx | 2 +- borg/key.py | 9 +-------- borg/remote.py | 2 +- borg/repository.py | 2 +- borg/xattr.py | 3 --- 5 files changed, 4 insertions(+), 14 deletions(-) diff --git a/borg/compress.pyx b/borg/compress.pyx index 2285b55d..3bb88def 100644 --- a/borg/compress.pyx +++ b/borg/compress.pyx @@ -110,7 +110,7 @@ cdef class LZ4(CompressorBase): class LZMA(CompressorBase): """ - lzma compression / decompression (python 3.3+ stdlib) + lzma compression / decompression """ ID = b'\x02\x00' name = 'lzma' diff --git a/borg/key.py b/borg/key.py index a39b1e31..7ee6305f 100644 --- a/borg/key.py +++ b/borg/key.py @@ -4,7 +4,7 @@ import getpass import os import sys import textwrap -import hmac +from hmac import HMAC from hashlib import sha256 from .helpers import IntegrityError, get_keys_dir, Error @@ -30,13 +30,6 @@ class RepoKeyNotFoundError(Error): """No key entry found in the config of repository {}.""" -class HMAC(hmac.HMAC): - """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews - """ - def update(self, msg): - self.inner.update(msg) - - def key_creator(repository, args): if args.encryption == 'keyfile': return KeyfileKey.create(repository, args) diff --git a/borg/remote.py b/borg/remote.py index d8092ec7..1f05d35e 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -268,7 +268,7 @@ class RemoteRepository: if not data: raise ConnectionClosed() data = data.decode('utf-8') - for line in data.splitlines(True): # keepends=True for py3.3+ + for line in data.splitlines(keepends=True): if line.startswith('$LOG '): _, level, msg = line.split(' ', 2) level = getattr(logging, level, logging.CRITICAL) # str -> int diff --git a/borg/repository.py b/borg/repository.py index 0789a021..1e5902f2 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -685,7 +685,7 @@ class LoggedIO: self.offset = 0 self._write_fd.flush() os.fsync(self._write_fd.fileno()) - if hasattr(os, 'posix_fadvise'): # python >= 3.3, only on UNIX + if hasattr(os, 'posix_fadvise'): # only on UNIX # tell the OS that it does not need to cache what we just wrote, # avoids spoiling the cache for the OS and other processes. os.posix_fadvise(self._write_fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED) diff --git a/borg/xattr.py b/borg/xattr.py index 7eafaa83..c3e85492 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -231,9 +231,6 @@ elif sys.platform.startswith('freebsd'): # pragma: freebsd only mv = memoryview(namebuf.raw) while mv: length = mv[0] - # Python < 3.3 returns bytes instead of int - if isinstance(length, bytes): - length = ord(length) names.append(os.fsdecode(bytes(mv[1:1+length]))) mv = mv[1+length:] return names From 19729d398329fb9688b3a81af887ead9787172a1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 14 Dec 2015 23:37:23 +0100 Subject: [PATCH 276/321] requirements: use latest pytest-cov, not blocked by py32 any more also: pytest-benchmark 3.x is released, just use latest --- requirements.d/development.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 7c1624b4..a0cb3c2a 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -1,6 +1,6 @@ virtualenv<14.0 tox pytest -pytest-cov<2.0.0 -pytest-benchmark>=3.0.0 +pytest-cov +pytest-benchmark Cython From a6f9c29dfeb2289394550f44ea661108a893aa67 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 00:09:36 +0100 Subject: [PATCH 277/321] use new OS and IO exception hierarchy of py 3.3 --- borg/archiver.py | 10 +++++----- borg/locking.py | 26 ++++++++++---------------- borg/repository.py | 2 +- borg/testsuite/archiver.py | 4 +--- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index e595715b..98f2a0e4 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -137,14 +137,14 @@ class Archiver: try: st = os.stat(get_cache_dir()) skip_inodes.add((st.st_ino, st.st_dev)) - except IOError: + except OSError: pass # Add local repository dir to inode_skip list if not args.location.host: try: st = os.stat(args.location.path) skip_inodes.add((st.st_ino, st.st_dev)) - except IOError: + except OSError: pass for path in args.paths: if path == '-': # stdin @@ -152,7 +152,7 @@ class Archiver: if not dry_run: try: status = archive.process_stdin(path, cache) - except IOError as e: + except OSError as e: status = 'E' self.print_warning('%s: %s', path, e) else: @@ -229,7 +229,7 @@ class Archiver: if not dry_run: try: status = archive.process_file(path, st, cache) - except IOError as e: + except OSError as e: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): @@ -326,7 +326,7 @@ class Archiver: archive.extract_item(item, restore_attrs=False) else: archive.extract_item(item, stdout=stdout, sparse=sparse) - except IOError as e: + except OSError as e: self.print_warning('%s: %s', remove_surrogates(orig_path), e) if not args.dry_run: diff --git a/borg/locking.py b/borg/locking.py index af17d8bc..dd7f96ff 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -132,14 +132,13 @@ class ExclusiveLock: while True: try: os.mkdir(self.path) + except FileExistsError: # already locked + if self.by_me(): + return self + if timer.timed_out_or_sleep(): + raise LockTimeout(self.path) except OSError as err: - if err.errno == errno.EEXIST: # already locked - if self.by_me(): - return self - if timer.timed_out_or_sleep(): - raise LockTimeout(self.path) - else: - raise LockFailed(self.path, str(err)) + raise LockFailed(self.path, str(err)) else: with open(self.unique_name, "wb"): pass @@ -181,12 +180,8 @@ class LockRoster: try: with open(self.path) as f: data = json.load(f) - except IOError as err: - if err.errno != errno.ENOENT: - raise - data = {} - except ValueError: - # corrupt/empty roster file? + except (FileNotFoundError, ValueError): + # no or corrupt/empty roster file? data = {} return data @@ -197,9 +192,8 @@ class LockRoster: def remove(self): try: os.unlink(self.path) - except OSError as e: - if e.errno != errno.ENOENT: - raise + except FileNotFoundError: + pass def get(self, key): roster = self.load() diff --git a/borg/repository.py b/borg/repository.py index 1e5902f2..0cf0d934 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -567,7 +567,7 @@ class LoggedIO: del self.fds[segment] try: os.unlink(self.segment_filename(segment)) - except OSError: + except FileNotFoundError: pass def segment_exists(self, segment): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 3192d60d..6bf2f635 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -70,9 +70,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): try: exec_cmd('help', exe='borg.exe', fork=True) BORG_EXES = ['python', 'binary', ] -except (IOError, OSError) as err: - if err.errno != errno.ENOENT: - raise +except FileNotFoundError: BORG_EXES = ['python', ] From fc52101d4680367eb87793ae0c25b09f537a254e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 00:17:03 +0100 Subject: [PATCH 278/321] suppress unneeded exception context (PEP 409) --- borg/archive.py | 4 ++-- borg/cache.py | 2 +- borg/fuse.py | 2 +- borg/helpers.py | 2 +- borg/key.py | 2 +- borg/locking.py | 2 +- borg/remote.py | 2 +- borg/repository.py | 6 +++--- borg/testsuite/__init__.py | 2 +- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index ca653ee8..5ece5724 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -292,7 +292,7 @@ Number of files: {0.stats.nfiles}'''.format(self) else: os.unlink(path) except UnicodeEncodeError: - raise self.IncompatibleFilesystemEncodingError(path, sys.getfilesystemencoding()) + raise self.IncompatibleFilesystemEncodingError(path, sys.getfilesystemencoding()) from None except OSError: pass mode = item[b'mode'] @@ -332,7 +332,7 @@ Number of files: {0.stats.nfiles}'''.format(self) try: os.symlink(source, path) except UnicodeEncodeError: - raise self.IncompatibleFilesystemEncodingError(source, sys.getfilesystemencoding()) + raise self.IncompatibleFilesystemEncodingError(source, sys.getfilesystemencoding()) from None self.restore_attrs(path, item, symlink=True) elif stat.S_ISFIFO(mode): if not os.path.exists(os.path.dirname(path)): diff --git a/borg/cache.py b/borg/cache.py index feffc1fd..707ad963 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -136,7 +136,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" raise Exception('%s has unexpected cache version %d (wanted: %d).' % ( config_path, cache_version, wanted_version)) except configparser.NoSectionError as e: - raise Exception('%s does not look like a Borg cache.' % config_path) + raise Exception('%s does not look like a Borg cache.' % config_path) from None self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) self.timestamp = self.config.get('cache', 'timestamp', fallback=None) diff --git a/borg/fuse.py b/borg/fuse.py index 36f761e4..a7c8845b 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -173,7 +173,7 @@ class FuseOperations(llfuse.Operations): try: return item.get(b'xattrs', {})[name] except KeyError: - raise llfuse.FUSEError(errno.ENODATA) + raise llfuse.FUSEError(errno.ENODATA) from None def _load_pending_archive(self, inode): # Check if this is an archive we need to load diff --git a/borg/helpers.py b/borg/helpers.py index 0a70fc63..43d034ae 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -777,7 +777,7 @@ def location_validator(archive=None): try: loc = Location(text) except ValueError: - raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) + raise argparse.ArgumentTypeError('Invalid location format: "%s"' % text) from None if archive is True and not loc.archive: raise argparse.ArgumentTypeError('"%s": No archive specified' % text) elif archive is False and loc.archive: diff --git a/borg/key.py b/borg/key.py index 7ee6305f..fbcc9d54 100644 --- a/borg/key.py +++ b/borg/key.py @@ -390,7 +390,7 @@ class RepoKey(KeyfileKeyBase): self.repository.load_key() return loc except configparser.NoOptionError: - raise RepoKeyNotFoundError(loc) + raise RepoKeyNotFoundError(loc) from None def get_new_target(self, args): return self.repository diff --git a/borg/locking.py b/borg/locking.py index dd7f96ff..99d20641 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -138,7 +138,7 @@ class ExclusiveLock: if timer.timed_out_or_sleep(): raise LockTimeout(self.path) except OSError as err: - raise LockFailed(self.path, str(err)) + raise LockFailed(self.path, str(err)) from None else: with open(self.unique_name, "wb"): pass diff --git a/borg/remote.py b/borg/remote.py index 1f05d35e..7de7f926 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -159,7 +159,7 @@ class RemoteRepository: try: version = self.call('negotiate', RPC_PROTOCOL_VERSION) except ConnectionClosed: - raise ConnectionClosedWithHint('Is borg working on the server?') + raise ConnectionClosedWithHint('Is borg working on the server?') from None if version != RPC_PROTOCOL_VERSION: raise Exception('Server insisted on using unsupported protocol version %d' % version) self.id = self.call('open', location.path, create, lock_wait, lock) diff --git a/borg/repository.py b/borg/repository.py index 0cf0d934..4a45fab2 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -417,7 +417,7 @@ class Repository: segment, offset = self.index[id_] return self.io.read(segment, offset, id_) except KeyError: - raise self.ObjectNotFound(id_, self.path) + raise self.ObjectNotFound(id_, self.path) from None def get_many(self, ids, is_preloaded=False): for id_ in ids: @@ -446,7 +446,7 @@ class Repository: try: segment, offset = self.index.pop(id) except KeyError: - raise self.ObjectNotFound(id, self.path) + raise self.ObjectNotFound(id, self.path) from None self.segments[segment] -= 1 self.compact.add(segment) segment = self.io.write_delete(id) @@ -628,7 +628,7 @@ class LoggedIO: hdr_tuple = fmt.unpack(header) except struct.error as err: raise IntegrityError('Invalid segment entry header [segment {}, offset {}]: {}'.format( - segment, offset, err)) + segment, offset, err)) from None if fmt is self.put_header_fmt: crc, size, tag, key = hdr_tuple elif fmt is self.header_fmt: diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 67c97789..1d09be50 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -144,4 +144,4 @@ class FakeInputs: try: return self.inputs.pop(0) except IndexError: - raise EOFError + raise EOFError from None From fc326df6001af5cf2f5497a54cacb93c01b67e33 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 00:26:06 +0100 Subject: [PATCH 279/321] a2b_base64 now also accepts ascii-only str objects --- borg/key.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/key.py b/borg/key.py index fbcc9d54..321158d1 100644 --- a/borg/key.py +++ b/borg/key.py @@ -263,7 +263,7 @@ class KeyfileKeyBase(AESKeyBase): raise NotImplementedError def _load(self, key_data, passphrase): - cdata = a2b_base64(key_data.encode('ascii')) # .encode needed for Python 3.[0-2] + cdata = a2b_base64(key_data) data = self.decrypt_key_file(cdata, passphrase) if data: key = msgpack.unpackb(data) From ef00f5d12de46616742dcbb9bf9419b3ad4a55b3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 00:41:50 +0100 Subject: [PATCH 280/321] we always have shutil.get_terminal_size on py 3.3+ --- borg/helpers.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 43d034ae..bbda5c60 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -6,12 +6,7 @@ import grp import os import pwd import re -try: - from shutil import get_terminal_size -except ImportError: - def get_terminal_size(fallback=(80, 24)): - TerminalSize = namedtuple('TerminalSize', ['columns', 'lines']) - return TerminalSize(int(os.environ.get('COLUMNS', fallback[0])), int(os.environ.get('LINES', fallback[1]))) +from shutil import get_terminal_size import sys import platform import time From 8e13d315bb2991ae7cc9337a6acefd018ffb7bea Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 01:00:28 +0100 Subject: [PATCH 281/321] use PyMemoryView_FromMemory (py 3.3+) --- borg/_chunker.c | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/borg/_chunker.c b/borg/_chunker.c index 75821493..396357c4 100644 --- a/borg/_chunker.c +++ b/borg/_chunker.c @@ -186,19 +186,6 @@ chunker_fill(Chunker *c) return 1; } -static PyObject * -PyBuffer_FromMemory(void *data, Py_ssize_t len) -{ - Py_buffer buffer; - PyObject *mv; - - PyBuffer_FillInfo(&buffer, NULL, data, len, 1, PyBUF_CONTIG_RO); - mv = PyMemoryView_FromBuffer(&buffer); - PyBuffer_Release(&buffer); - return mv; -} - - static PyObject * chunker_process(Chunker *c) { @@ -221,7 +208,7 @@ chunker_process(Chunker *c) c->done = 1; if(c->remaining) { c->bytes_yielded += c->remaining; - return PyBuffer_FromMemory(c->data + c->position, c->remaining); + return PyMemoryView_FromMemory(c->data + c->position, c->remaining, PyBUF_READ); } else { if(c->bytes_read == c->bytes_yielded) @@ -253,5 +240,5 @@ chunker_process(Chunker *c) c->last = c->position; n = c->last - old_last; c->bytes_yielded += n; - return PyBuffer_FromMemory(c->data + old_last, n); + return PyMemoryView_FromMemory(c->data + old_last, n, PyBUF_READ); } From 0be62d423331bd8e44172f596c21aabf488ff790 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 01:08:35 +0100 Subject: [PATCH 282/321] stuff found on "Porting to Python 3.3" --- borg/archiver.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 98f2a0e4..a75bef9e 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -438,7 +438,7 @@ class Archiver: pass try: mtime = datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9) - except ValueError: + except OverflowError: # likely a broken mtime and datetime did not want to go beyond year 9999 mtime = datetime(9999, 12, 31, 23, 59, 59) if b'source' in item: diff --git a/setup.py b/setup.py index b7a43daa..2ba2de47 100644 --- a/setup.py +++ b/setup.py @@ -211,7 +211,7 @@ if not on_rtd: Extension('borg.chunker', [chunker_source]), Extension('borg.hashindex', [hashindex_source]) ] - if sys.platform.startswith('linux'): + if sys.platform == 'linux': ext_modules.append(Extension('borg.platform_linux', [platform_linux_source], libraries=['acl'])) elif sys.platform.startswith('freebsd'): ext_modules.append(Extension('borg.platform_freebsd', [platform_freebsd_source])) From 7c8bfe6681866781872f48fb5fd7d3478fce184b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 01:25:25 +0100 Subject: [PATCH 283/321] __file__ is now always an absolute path (3.4) --- borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6bf2f635..fae8a46f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -34,7 +34,7 @@ except ImportError: has_lchflags = hasattr(os, 'lchflags') -src_dir = os.path.join(os.getcwd(), os.path.dirname(__file__), '..') +src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): From 9fa18c9ee97000fe343a2a47db6d928f36376d34 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 18:46:25 +0100 Subject: [PATCH 284/321] use stat.filemode instead of homegrown code --- borg/archiver.py | 13 ++++++------- borg/helpers.py | 9 --------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index a75bef9e..eff722b7 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -15,7 +15,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ - format_file_mode, parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ + parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, is_slow_msgpack, yes, sysinfo, \ @@ -426,10 +426,9 @@ class Archiver: for item in archive.iter_items(): print(remove_surrogates(item[b'path'])) else: - tmap = {1: 'p', 2: 'c', 4: 'd', 6: 'b', 0o10: '-', 0o12: 'l', 0o14: 's'} for item in archive.iter_items(): - type = tmap.get(item[b'mode'] // 4096, '?') - mode = format_file_mode(item[b'mode']) + mode = stat.filemode(item[b'mode']) + type = mode[0] size = 0 if type == '-': try: @@ -445,12 +444,12 @@ class Archiver: if type == 'l': extra = ' -> %s' % item[b'source'] else: - type = 'h' + mode = 'h' + mode[1:] extra = ' link to %s' % item[b'source'] else: extra = '' - print('%s%s %-6s %-6s %8d %s %s%s' % ( - type, mode, item[b'user'] or item[b'uid'], + print('%s %-6s %-6s %8d %s %s%s' % ( + mode, item[b'user'] or item[b'uid'], item[b'group'] or item[b'gid'], size, format_time(mtime), remove_surrogates(item[b'path']), extra)) else: diff --git a/borg/helpers.py b/borg/helpers.py index bbda5c60..3bb7cf5b 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -558,15 +558,6 @@ def format_timedelta(td): return txt -def format_file_mode(mod): - """Format file mode bits for list output - """ - def x(v): - return ''.join(v & m and s or '-' - for m, s in ((4, 'r'), (2, 'w'), (1, 'x'))) - return '%s%s%s' % (x(mod // 64), x(mod // 8), x(mod)) - - def format_file_size(v, precision=2): """Format file size into a human friendly format """ From 2cc02255272160118dfe24d90bb6a7d844a0355a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 19:01:52 +0100 Subject: [PATCH 285/321] use hmac.compare_digest instead of == operator this is available in python 3.3+ --- borg/key.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/borg/key.py b/borg/key.py index 321158d1..97a8dca5 100644 --- a/borg/key.py +++ b/borg/key.py @@ -4,7 +4,7 @@ import getpass import os import sys import textwrap -from hmac import HMAC +from hmac import HMAC, compare_digest from hashlib import sha256 from .helpers import IntegrityError, get_keys_dir, Error @@ -134,13 +134,17 @@ class AESKeyBase(KeyBase): def decrypt(self, id, data): if data[0] != self.TYPE: raise IntegrityError('Invalid encryption envelope') - hmac = memoryview(data)[1:33] - if memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) != hmac: + hmac_given = memoryview(data)[1:33] + hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) + if not compare_digest(hmac_computed, hmac_given): raise IntegrityError('Encryption envelope checksum mismatch') self.dec_cipher.reset(iv=PREFIX + data[33:41]) data = self.compressor.decompress(self.dec_cipher.decrypt(data[41:])) - if id and HMAC(self.id_key, data, sha256).digest() != id: - raise IntegrityError('Chunk id verification failed') + if id: + hmac_given = id + hmac_computed = HMAC(self.id_key, data, sha256).digest() + if not compare_digest(hmac_computed, hmac_given): + raise IntegrityError('Chunk id verification failed') return data def extract_nonce(self, payload): From 3ade3d8a417cab9e0bde88ee7b10cf6bce2df7c1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2015 19:48:40 +0100 Subject: [PATCH 286/321] use hashlib.pbkdf2_hmac from py stdlib instead of own openssl wrapper this is available in python 3.4+. note: before removing the pbkdf tests, i ran them with the pbkdf from stdlib to make sure it gives same result. long term testing of this now belongs into stdlib tests, not into borg. --- borg/crypto.pyx | 23 +---------------------- borg/key.py | 6 +++--- borg/testsuite/crypto.py | 10 +--------- 3 files changed, 5 insertions(+), 34 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index d8143bdb..199ffbaf 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -1,7 +1,6 @@ """A thin OpenSSL wrapper -This could be replaced by PyCrypto or something similar when the performance -of their PBKDF2 implementation is comparable to the OpenSSL version. +This could be replaced by PyCrypto maybe? """ from libc.stdlib cimport malloc, free @@ -21,7 +20,6 @@ cdef extern from "openssl/evp.h": pass ctypedef struct ENGINE: pass - const EVP_MD *EVP_sha256() const EVP_CIPHER *EVP_aes_256_ctr() void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a) void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a) @@ -37,10 +35,6 @@ cdef extern from "openssl/evp.h": int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) - int PKCS5_PBKDF2_HMAC(const char *password, int passwordlen, - const unsigned char *salt, int saltlen, int iter, - const EVP_MD *digest, - int keylen, unsigned char *out) import struct @@ -59,21 +53,6 @@ def num_aes_blocks(int length): return (length + 15) // 16 -def pbkdf2_sha256(password, salt, iterations, size): - """Password based key derivation function 2 (RFC2898) - """ - cdef unsigned char *key = malloc(size) - if not key: - raise MemoryError - try: - rv = PKCS5_PBKDF2_HMAC(password, len(password), salt, len(salt), iterations, EVP_sha256(), size, key) - if not rv: - raise Exception('PKCS5_PBKDF2_HMAC failed') - return key[:size] - finally: - free(key) - - def get_random_bytes(n): """Return n cryptographically strong pseudo-random bytes """ diff --git a/borg/key.py b/borg/key.py index 97a8dca5..fe3b1d5a 100644 --- a/borg/key.py +++ b/borg/key.py @@ -5,13 +5,13 @@ import os import sys import textwrap from hmac import HMAC, compare_digest -from hashlib import sha256 +from hashlib import sha256, pbkdf2_hmac from .helpers import IntegrityError, get_keys_dir, Error from .logger import create_logger logger = create_logger() -from .crypto import pbkdf2_sha256, get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks +from .crypto import get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from .compress import Compressor, COMPR_BUFFER import msgpack @@ -199,7 +199,7 @@ class Passphrase(str): return '' def kdf(self, salt, iterations, length): - return pbkdf2_sha256(self.encode('utf-8'), salt, iterations, length) + return pbkdf2_hmac('sha256', self.encode('utf-8'), salt, iterations, length) class PassphraseKey(AESKeyBase): diff --git a/borg/testsuite/crypto.py b/borg/testsuite/crypto.py index e438eb85..e8f56515 100644 --- a/borg/testsuite/crypto.py +++ b/borg/testsuite/crypto.py @@ -1,6 +1,6 @@ from binascii import hexlify -from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, pbkdf2_sha256, get_random_bytes +from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, get_random_bytes from . import BaseTestCase @@ -13,14 +13,6 @@ class CryptoTestCase(BaseTestCase): self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') - def test_pbkdf2_sha256(self): - self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 1, 32)), - b'120fb6cffcf8b32c43e7225256c4f837a86548c92ccc35480805987cb70be17b') - self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 2, 32)), - b'ae4d0c95af6b46d32d0adff928f06dd02a303f8ef3c251dfd6e2d85a95474c43') - self.assert_equal(hexlify(pbkdf2_sha256(b'password', b'salt', 4096, 32)), - b'c5e478d59288c841aa530db6845c4c8d962893a001ce4e11a4963873aa98134a') - def test_get_random_bytes(self): bytes = get_random_bytes(10) bytes2 = get_random_bytes(10) From 5607e5aefe81c2d0808cea14eee098f4265cb9a8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 18 Dec 2015 21:05:59 +0100 Subject: [PATCH 287/321] use os.urandom instead of own cython openssl RAND_bytes wrapper, fixes #493 --- borg/crypto.pyx | 14 -------------- borg/key.py | 6 +++--- borg/testsuite/benchmark.py | 5 +---- borg/testsuite/crypto.py | 9 +-------- 4 files changed, 5 insertions(+), 29 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 199ffbaf..172fe074 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -53,20 +53,6 @@ def num_aes_blocks(int length): return (length + 15) // 16 -def get_random_bytes(n): - """Return n cryptographically strong pseudo-random bytes - """ - cdef unsigned char *buf = malloc(n) - if not buf: - raise MemoryError - try: - if RAND_bytes(buf, n) < 1: - raise Exception('RAND_bytes failed') - return buf[:n] - finally: - free(buf) - - cdef class AES: """A thin wrapper around the OpenSSL EVP cipher API """ diff --git a/borg/key.py b/borg/key.py index fe3b1d5a..70999fb2 100644 --- a/borg/key.py +++ b/borg/key.py @@ -11,7 +11,7 @@ from .helpers import IntegrityError, get_keys_dir, Error from .logger import create_logger logger = create_logger() -from .crypto import get_random_bytes, AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks +from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks from .compress import Compressor, COMPR_BUFFER import msgpack @@ -291,7 +291,7 @@ class KeyfileKeyBase(AESKeyBase): return data def encrypt_key_file(self, data, passphrase): - salt = get_random_bytes(32) + salt = os.urandom(32) iterations = 100000 key = passphrase.kdf(salt, iterations, 32) hash = HMAC(key, data, sha256).digest() @@ -329,7 +329,7 @@ class KeyfileKeyBase(AESKeyBase): passphrase = Passphrase.new(allow_empty=True) key = cls(repository) key.repository_id = repository.id - key.init_from_random_data(get_random_bytes(100)) + key.init_from_random_data(os.urandom(100)) key.init_ciphers() target = key.get_new_target(args) key.save(target, passphrase) diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py index 3d59b672..6979fcfa 100644 --- a/borg/testsuite/benchmark.py +++ b/borg/testsuite/benchmark.py @@ -40,13 +40,10 @@ def testdata(request, tmpdir_factory): # do not use a binary zero (\0) to avoid sparse detection data = lambda: b'0' * size if data_type == 'random': - rnd = open('/dev/urandom', 'rb') - data = lambda: rnd.read(size) + data = lambda: os.urandom(size) for i in range(count): with open(str(p.join(str(i))), "wb") as f: f.write(data()) - if data_type == 'random': - rnd.close() yield str(p) p.remove(rec=1) diff --git a/borg/testsuite/crypto.py b/borg/testsuite/crypto.py index e8f56515..2d74493d 100644 --- a/borg/testsuite/crypto.py +++ b/borg/testsuite/crypto.py @@ -1,6 +1,6 @@ from binascii import hexlify -from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, get_random_bytes +from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes from . import BaseTestCase @@ -13,13 +13,6 @@ class CryptoTestCase(BaseTestCase): self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') - def test_get_random_bytes(self): - bytes = get_random_bytes(10) - bytes2 = get_random_bytes(10) - self.assert_equal(len(bytes), 10) - self.assert_equal(len(bytes2), 10) - self.assert_not_equal(bytes, bytes2) - def test_aes(self): key = b'X' * 32 data = b'foo' * 10 From 169634f2ca891a356e83062f130da24d5a719dc0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 18 Dec 2015 21:44:14 +0100 Subject: [PATCH 288/321] change the builtin default for --chunker-params, create 2MiB chunks, fixes #343 one of the biggest issues with borg < 1.0 was that it had a default target chunk size of 64kiB, thus it created a lot of chunks, a huge chunk management overhead (high RAM and disk usage). --- borg/archive.py | 4 ++-- docs/internals.rst | 10 +++++----- docs/misc/create_chunker-params.txt | 4 ++-- docs/usage.rst | 22 ++++++++++++---------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 5ece5724..ad22d3d1 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -26,10 +26,10 @@ import msgpack ITEMS_BUFFER = 1024 * 1024 -CHUNK_MIN_EXP = 10 # 2**10 == 1kiB +CHUNK_MIN_EXP = 19 # 2**19 == 512kiB CHUNK_MAX_EXP = 23 # 2**23 == 8MiB HASH_WINDOW_SIZE = 0xfff # 4095B -HASH_MASK_BITS = 16 # results in ~64kiB chunks statistically +HASH_MASK_BITS = 21 # results in ~2MiB chunks statistically # defaults, use --chunker-params to override CHUNKER_PARAMS = (CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE) diff --git a/docs/internals.rst b/docs/internals.rst index 246ffca9..21ae2445 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -210,9 +210,9 @@ producing chunks of 2^HASH_MASK_BITS Bytes on average. ``borg create --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE`` can be used to tune the chunker parameters, the default is: -- CHUNK_MIN_EXP = 10 (minimum chunk size = 2^10 B = 1 kiB) +- CHUNK_MIN_EXP = 19 (minimum chunk size = 2^19 B = 512 kiB) - CHUNK_MAX_EXP = 23 (maximum chunk size = 2^23 B = 8 MiB) -- HASH_MASK_BITS = 16 (statistical medium chunk size ~= 2^16 B = 64 kiB) +- HASH_MASK_BITS = 21 (statistical medium chunk size ~= 2^21 B = 2 MiB) - HASH_WINDOW_SIZE = 4095 [B] (`0xFFF`) The buzhash table is altered by XORing it with a seed randomly generated once @@ -313,13 +313,13 @@ If a remote repository is used the repo index will be allocated on the remote si E.g. backing up a total count of 1 Mi (IEC binary prefix e.g. 2^20) files with a total size of 1TiB. -a) with create ``--chunker-params 10,23,16,4095`` (default): +a) with ``create --chunker-params 10,23,16,4095`` (custom, like borg < 1.0 or attic): mem_usage = 2.8GiB -b) with create ``--chunker-params 10,23,20,4095`` (custom): +b) with ``create --chunker-params 19,23,21,4095`` (default): - mem_usage = 0.4GiB + mem_usage = 0.31GiB .. note:: There is also the ``--no-files-cache`` option to switch off the files cache. You'll save some memory, but it will need to read / chunk all the files as diff --git a/docs/misc/create_chunker-params.txt b/docs/misc/create_chunker-params.txt index 73cac6a3..3e322b66 100644 --- a/docs/misc/create_chunker-params.txt +++ b/docs/misc/create_chunker-params.txt @@ -6,7 +6,7 @@ About borg create --chunker-params CHUNK_MIN_EXP and CHUNK_MAX_EXP give the exponent N of the 2^N minimum and maximum chunk size. Required: CHUNK_MIN_EXP < CHUNK_MAX_EXP. -Defaults: 10 (2^10 == 1KiB) minimum, 23 (2^23 == 8MiB) maximum. +Defaults: 19 (2^19 == 512KiB) minimum, 23 (2^23 == 8MiB) maximum. HASH_MASK_BITS is the number of least-significant bits of the rolling hash that need to be zero to trigger a chunk cut. @@ -14,7 +14,7 @@ Recommended: CHUNK_MIN_EXP + X <= HASH_MASK_BITS <= CHUNK_MAX_EXP - X, X >= 2 (this allows the rolling hash some freedom to make its cut at a place determined by the windows contents rather than the min/max. chunk size). -Default: 16 (statistically, chunks will be about 2^16 == 64kiB in size) +Default: 21 (statistically, chunks will be about 2^21 == 2MiB in size) HASH_WINDOW_SIZE: the size of the window used for the rolling hash computation. Default: 4095B diff --git a/docs/usage.rst b/docs/usage.rst index 4abd44b2..d6aad6a8 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -249,8 +249,10 @@ Examples NAME="root-`date +%Y-%m-%d`" $ borg create -C zlib,6 /mnt/backup::$NAME / --do-not-cross-mountpoints - # Backup huge files with little chunk management overhead - $ borg create --chunker-params 19,23,21,4095 /mnt/backup::VMs /srv/VMs + # Make a big effort in fine granular deduplication (big chunk management + # overhead, needs a lot of RAM and disk space, see formula in internals + # docs - same parameters as borg < 1.0 or attic): + $ borg create --chunker-params 10,23,16,4095 /mnt/backup::small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) $ dd if=/dev/sda bs=10M | borg create /mnt/backup::my-sda - @@ -506,15 +508,15 @@ resource usage (RAM and disk space) as the amount of resources needed is (also) determined by the total amount of chunks in the repository (see `Indexes / Caches memory usage` for details). -``--chunker-params=10,23,16,4095 (default)`` results in a fine-grained deduplication -and creates a big amount of chunks and thus uses a lot of resources to manage them. -This is good for relatively small data volumes and if the machine has a good -amount of free RAM and disk space. +``--chunker-params=10,23,16,4095`` results in a fine-grained deduplication +and creates a big amount of chunks and thus uses a lot of resources to manage +them. This is good for relatively small data volumes and if the machine has a +good amount of free RAM and disk space. -``--chunker-params=19,23,21,4095`` results in a coarse-grained deduplication and -creates a much smaller amount of chunks and thus uses less resources. -This is good for relatively big data volumes and if the machine has a relatively -low amount of free RAM and disk space. +``--chunker-params=19,23,21,4095`` (default) results in a coarse-grained +deduplication and creates a much smaller amount of chunks and thus uses less +resources. This is good for relatively big data volumes and if the machine has +a relatively low amount of free RAM and disk space. If you already have made some archives in a repository and you then change chunker params, this of course impacts deduplication as the chunks will be From 6d615ec30a2f4ae0ae45c34e753fe5da848948e4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 19 Dec 2015 14:30:05 +0100 Subject: [PATCH 289/321] change encryption to be on by default (repokey mode) it's 2015, let's be safe-by-default and unsafe-as-option. also: show default mode in builtin help --- borg/archiver.py | 5 ++--- borg/remote.py | 2 +- borg/repository.py | 3 ++- borg/testsuite/archiver.py | 3 ++- docs/quickstart.rst | 5 +++-- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index eff722b7..69bda26a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -64,7 +64,6 @@ class Archiver: repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args) else: repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait, lock=lock) - repository._location = location return repository def print_error(self, msg, *args): @@ -797,8 +796,8 @@ class Archiver: type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', - choices=('none', 'keyfile', 'repokey', 'passphrase'), default='none', - help='select encryption key mode') + choices=('none', 'keyfile', 'repokey', 'passphrase'), default='repokey', + help='select encryption key mode (default: "%(default)s")') check_epilog = textwrap.dedent(""" The check command verifies the consistency of a repository and the corresponding archives. diff --git a/borg/remote.py b/borg/remote.py index 7de7f926..a836ac2c 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -129,7 +129,7 @@ class RemoteRepository: self.name = name def __init__(self, location, create=False, lock_wait=None, lock=True, args=None): - self.location = location + self.location = self._location = location self.preload_ids = [] self.msgid = 0 self.to_send = b'' diff --git a/borg/repository.py b/borg/repository.py index 4a45fab2..334065bc 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -11,7 +11,7 @@ import struct from zlib import crc32 import msgpack -from .helpers import Error, ErrorWithTraceback, IntegrityError, ProgressIndicatorPercent +from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache @@ -54,6 +54,7 @@ class Repository: def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=True): self.path = os.path.abspath(path) + self._location = Location('file://%s' % self.path) self.io = None self.lock = None self.index = None diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index fae8a46f..90d85c05 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -92,7 +92,7 @@ def test_return_codes(cmd, tmpdir): input = tmpdir.mkdir('input') output = tmpdir.mkdir('output') input.join('test_file').write('content') - rc, out = cmd('init', '%s' % str(repo)) + rc, out = cmd('init', '--encryption=none', '%s' % str(repo)) assert rc == EXIT_SUCCESS rc, out = cmd('create', '%s::archive' % repo, str(input)) assert rc == EXIT_SUCCESS @@ -192,6 +192,7 @@ class ArchiverTestCaseBase(BaseTestCase): def setUp(self): os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1' os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1' + os.environ['BORG_PASSPHRASE'] = 'waytooeasyonlyfortests' self.archiver = not self.FORK_DEFAULT and Archiver() or None self.tmpdir = tempfile.mkdtemp() self.repository_path = os.path.join(self.tmpdir, 'repository') diff --git a/docs/quickstart.rst b/docs/quickstart.rst index ca7acc79..57e5481f 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -146,9 +146,10 @@ Keep an eye on CPU load and throughput. Repository encryption --------------------- -Repository encryption is enabled at repository creation time:: +Repository encryption can be enabled or disabled at repository creation time +(the default is enabled, with `repokey` method):: - $ borg init --encryption=repokey|keyfile PATH + $ borg init --encryption=none|repokey|keyfile PATH When repository encryption is enabled all data is encrypted using 256-bit AES_ encryption and the integrity and authenticity is verified using `HMAC-SHA256`_. From b2dedee3c821415d05996dc6414254610bc14040 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 20 Dec 2015 01:34:00 +0100 Subject: [PATCH 290/321] refactor yes(), cleanup env var semantics, fixes #355 refactorings: - introduced concept of default answer: if the answer string is in the defaultish sequence, the return value of yes() will be the default. e.g. if just pressing when asked on the console or if an empty string or "default" is in the environment variable for overriding. if an environment var has an invalid value and no retries are enabled: return default if retries are enabled, next retry won't use the env var again, but either ask via input(). - simplify: only one default - this should be a SAFE default as it is used in some special conditions like EOF or invalid input with retries disallowed. no isatty() magic, the "yes" shell command exists, so we could receive input even if it is not from a tty. - clean: separate retry flag from retry_msg --- borg/archiver.py | 8 ++-- borg/cache.py | 6 +-- borg/helpers.py | 86 +++++++++++++++++++------------------ borg/testsuite/archiver.py | 8 ++-- borg/testsuite/benchmark.py | 6 +-- borg/testsuite/helpers.py | 81 ++++++++++++++++++++-------------- docs/usage.rst | 18 +++++--- 7 files changed, 117 insertions(+), 96 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 69bda26a..5afb2f89 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -105,8 +105,8 @@ class Archiver: msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + "Type 'YES' if you understand this and want to continue: ") - if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + if not yes(msg, false_msg="Aborting.", truish=('YES', ), + env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): return EXIT_ERROR if not args.archives_only: if not repository.check(repair=args.repair, save_space=args.save_space): @@ -374,8 +374,8 @@ class Archiver: msg.append(format_archive(archive_info)) msg.append("Type 'YES' if you understand this and want to continue: ") msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): + if not yes(msg, false_msg="Aborting.", truish=('YES', ), + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): self.exit_code = EXIT_ERROR return self.exit_code repository.destroy() diff --git a/borg/cache.py b/borg/cache.py index 707ad963..521b6846 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -54,8 +54,7 @@ class Cache: msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): + if not yes(msg, false_msg="Aborting.", env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): raise self.CacheInitAbortedError() self.create() self.open(lock_wait=lock_wait) @@ -64,8 +63,7 @@ class Cache: msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", default_notty=False, - env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): + if not yes(msg, false_msg="Aborting.", env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): raise self.RepositoryAccessAborted() if sync and self.manifest.id != self.manifest_id: diff --git a/borg/helpers.py b/borg/helpers.py index 3bb7cf5b..7957ee38 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -842,71 +842,69 @@ def is_slow_msgpack(): return msgpack.Packer is msgpack.fallback.Packer -def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, - default=False, default_notty=None, default_eof=None, - falsish=('No', 'no', 'N', 'n'), truish=('Yes', 'yes', 'Y', 'y'), - env_var_override=None, ifile=None, ofile=None, input=input): +FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', ) +TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', ) +DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', ) + +def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, + retry_msg=None, invalid_msg=None, env_msg=None, + falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH, + default=False, retry=True, env_var_override=None, ofile=None, input=input): """ Output (usually a question) and let user input an answer. - Qualifies the answer according to falsish and truish as True or False. + Qualifies the answer according to falsish, truish and defaultish as True, False or . If it didn't qualify and retry_msg is None (no retries wanted), return the default [which defaults to False]. Otherwise let user retry answering until answer is qualified. - If env_var_override is given and it is non-empty, counts as truish answer - and won't ask user for an answer. - If we don't have a tty as input and default_notty is not None, return its value. - Otherwise read input from non-tty and proceed as normal. - If EOF is received instead an input, return default_eof [or default, if not given]. + If env_var_override is given and this var is present in the environment, do not ask + the user, but just use the env var contents as answer as if it was typed in. + Otherwise read input from stdin and proceed as normal. + If EOF is received instead an input or an invalid input without retry possibility, + return default. :param msg: introducing message to output on ofile, no \n is added [None] :param retry_msg: retry message to output on ofile, no \n is added [None] - (also enforces retries instead of returning default) :param false_msg: message to output before returning False [None] :param true_msg: message to output before returning True [None] - :param default: default return value (empty answer is given) [False] - :param default_notty: if not None, return its value if no tty is connected [None] - :param default_eof: return value if EOF was read as answer [same as default] + :param default_msg: message to output before returning a [None] + :param invalid_msg: message to output after a invalid answer was given [None] + :param env_msg: message to output when using input from env_var_override [None], + needs to have 2 placeholders for answer and env var name, e.g.: "{} (from {})" :param falsish: sequence of answers qualifying as False :param truish: sequence of answers qualifying as True + :param defaultish: sequence of answers qualifying as + :param default: default return value (defaultish answer was given or no-answer condition) [False] + :param retry: if True and input is incorrect, retry. Otherwise return default. [True] :param env_var_override: environment variable name [None] - :param ifile: input stream [sys.stdin] (only for testing!) :param ofile: output stream [sys.stderr] :param input: input function [input from builtins] :return: boolean answer value, True or False """ - # note: we do not assign sys.stdin/stderr as defaults above, so they are + # note: we do not assign sys.stderr as default above, so it is # really evaluated NOW, not at function definition time. - if ifile is None: - ifile = sys.stdin if ofile is None: ofile = sys.stderr if default not in (True, False): raise ValueError("invalid default value, must be True or False") - if default_notty not in (None, True, False): - raise ValueError("invalid default_notty value, must be None, True or False") - if default_eof not in (None, True, False): - raise ValueError("invalid default_eof value, must be None, True or False") if msg: - print(msg, file=ofile, end='') - ofile.flush() - if env_var_override: - value = os.environ.get(env_var_override) - # currently, any non-empty value counts as truish - # TODO: change this so one can give y/n there? - if value: - value = bool(value) - value_str = truish[0] if value else falsish[0] - print("{} (from {})".format(value_str, env_var_override), file=ofile) - return value - if default_notty is not None and not ifile.isatty(): - # looks like ifile is not a terminal (but e.g. a pipe) - return default_notty + print(msg, file=ofile, end='', flush=True) while True: - try: - answer = input() # XXX how can we use ifile? - except EOFError: - return default_eof if default_eof is not None else default + answer = None + if env_var_override: + answer = os.environ.get(env_var_override) + if answer is not None and env_msg: + print(env_msg.format(answer, env_var_override), file=ofile) + if answer is None: + try: + answer = input() + except EOFError: + # avoid defaultish[0], defaultish could be empty + answer = truish[0] if default else falsish[0] + if answer in defaultish: + if default_msg: + print(default_msg, file=ofile) + return default if answer in truish: if true_msg: print(true_msg, file=ofile) @@ -915,11 +913,15 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, if false_msg: print(false_msg, file=ofile) return False - if retry_msg is None: - # no retries wanted, we just return the default + # if we get here, the answer was invalid + if invalid_msg: + print(invalid_msg, file=ofile) + if not retry: return default if retry_msg: print(retry_msg, file=ofile, end='', flush=True) + # in case we used an environment variable and it gave an invalid answer, do not use it again: + env_var_override = None class ProgressIndicatorPercent: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 90d85c05..ea9d1a72 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -136,7 +136,7 @@ def test_disk_full(cmd): data = os.urandom(size) f.write(data) - with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='1'): + with environment_variable(BORG_CHECK_I_KNOW_WHAT_I_AM_DOING='YES'): mount = DF_MOUNT assert os.path.exists(mount) repo = os.path.join(mount, 'repo') @@ -190,8 +190,8 @@ class ArchiverTestCaseBase(BaseTestCase): prefix = '' def setUp(self): - os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1' - os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1' + os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES' + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' os.environ['BORG_PASSPHRASE'] = 'waytooeasyonlyfortests' self.archiver = not self.FORK_DEFAULT and Archiver() or None self.tmpdir = tempfile.mkdtemp() @@ -330,7 +330,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): item_count = 3 if has_lchflags else 4 # one file is UF_NODUMP self.assert_in('Number of files: %d' % item_count, info_output) shutil.rmtree(self.cache_path) - with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='1'): + with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): info_output2 = self.cmd('info', self.repository_location + '::test') def filter(output): diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py index 6979fcfa..cb7f9e90 100644 --- a/borg/testsuite/benchmark.py +++ b/borg/testsuite/benchmark.py @@ -16,9 +16,9 @@ from .archiver import changedir, cmd @pytest.yield_fixture def repo_url(request, tmpdir): os.environ['BORG_PASSPHRASE'] = '123456' - os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1' - os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = '1' - os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1' + os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = 'YES' + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' + os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = 'yes' os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys')) os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache')) yield str(tmpdir.join('repository')) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index cc4b3df3..82b3f135 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -9,11 +9,11 @@ import sys import msgpack import msgpack.fallback -from ..helpers import Location, format_file_size, format_timedelta, PathPrefixPattern, FnmatchPattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, RegexPattern, \ +from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \ + prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, TRUISH, FALSISH, DEFAULTISH, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ - ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, PatternMatcher, \ - ShellPattern + ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \ + PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern from . import BaseTestCase, environment_variable, FakeInputs @@ -691,20 +691,28 @@ def test_is_slow_msgpack(): assert not is_slow_msgpack() -def test_yes_simple(): - input = FakeInputs(['y', 'Y', 'yes', 'Yes', ]) - assert yes(input=input) - assert yes(input=input) - assert yes(input=input) - assert yes(input=input) - input = FakeInputs(['n', 'N', 'no', 'No', ]) - assert not yes(input=input) - assert not yes(input=input) - assert not yes(input=input) - assert not yes(input=input) +def test_yes_input(): + inputs = list(TRUISH) + input = FakeInputs(inputs) + for i in inputs: + assert yes(input=input) + inputs = list(FALSISH) + input = FakeInputs(inputs) + for i in inputs: + assert not yes(input=input) -def test_yes_custom(): +def test_yes_input_defaults(): + inputs = list(DEFAULTISH) + input = FakeInputs(inputs) + for i in inputs: + assert yes(default=True, input=input) + input = FakeInputs(inputs) + for i in inputs: + assert not yes(default=False, input=input) + + +def test_yes_input_custom(): input = FakeInputs(['YES', 'SURE', 'NOPE', ]) assert yes(truish=('YES', ), input=input) assert yes(truish=('SURE', ), input=input) @@ -712,11 +720,20 @@ def test_yes_custom(): def test_yes_env(): - input = FakeInputs(['n', 'n']) - with environment_variable(OVERRIDE_THIS='nonempty'): - assert yes(env_var_override='OVERRIDE_THIS', input=input) - with environment_variable(OVERRIDE_THIS=None): # env not set - assert not yes(env_var_override='OVERRIDE_THIS', input=input) + for value in TRUISH: + with environment_variable(OVERRIDE_THIS=value): + assert yes(env_var_override='OVERRIDE_THIS') + for value in FALSISH: + with environment_variable(OVERRIDE_THIS=value): + assert not yes(env_var_override='OVERRIDE_THIS') + + +def test_yes_env_default(): + for value in DEFAULTISH: + with environment_variable(OVERRIDE_THIS=value): + assert yes(env_var_override='OVERRIDE_THIS', default=True) + with environment_variable(OVERRIDE_THIS=value): + assert not yes(env_var_override='OVERRIDE_THIS', default=False) def test_yes_defaults(): @@ -728,27 +745,27 @@ def test_yes_defaults(): assert yes(default=True, input=input) assert yes(default=True, input=input) assert yes(default=True, input=input) - ifile = StringIO() - assert yes(default_notty=True, ifile=ifile) - assert not yes(default_notty=False, ifile=ifile) input = FakeInputs([]) - assert yes(default_eof=True, input=input) - assert not yes(default_eof=False, input=input) + assert yes(default=True, input=input) + assert not yes(default=False, input=input) with pytest.raises(ValueError): yes(default=None) - with pytest.raises(ValueError): - yes(default_notty='invalid') - with pytest.raises(ValueError): - yes(default_eof='invalid') def test_yes_retry(): - input = FakeInputs(['foo', 'bar', 'y', ]) + input = FakeInputs(['foo', 'bar', TRUISH[0], ]) assert yes(retry_msg='Retry: ', input=input) - input = FakeInputs(['foo', 'bar', 'N', ]) + input = FakeInputs(['foo', 'bar', FALSISH[0], ]) assert not yes(retry_msg='Retry: ', input=input) +def test_yes_no_retry(): + input = FakeInputs(['foo', 'bar', TRUISH[0], ]) + assert not yes(retry=False, default=False, input=input) + input = FakeInputs(['foo', 'bar', FALSISH[0], ]) + assert yes(retry=False, default=True, input=input) + + def test_yes_output(capfd): input = FakeInputs(['invalid', 'y', 'n']) assert yes(msg='intro-msg', false_msg='false-msg', true_msg='true-msg', retry_msg='retry-msg', input=input) diff --git a/docs/usage.rst b/docs/usage.rst index d6aad6a8..3094f403 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -69,15 +69,19 @@ General: TMPDIR where temporary files are stored (might need a lot of temporary space for some operations) -Some "yes" sayers (if set, they automatically confirm that you really want to do X even if there is that warning): - BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK +Some automatic "answerers" (if set, they automatically answer confirmation questions): + BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes) For "Warning: Attempting to access a previously unknown unencrypted repository" - BORG_RELOCATED_REPO_ACCESS_IS_OK + BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes) For "Warning: The repository at location ... was previously located at ..." - BORG_CHECK_I_KNOW_WHAT_I_AM_DOING - For "Warning: '``check --repair``' is an experimental feature that might result in data loss." - BORG_DELETE_I_KNOW_WHAT_I_AM_DOING - For "You requested to completely DELETE the repository *including* all archives it contains: " + BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) + For "Warning: 'check --repair' is an experimental feature that might result in data loss." + BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) + For "You requested to completely DELETE the repository *including* all archives it contains:" + + Note: answers are case sensitive. setting an invalid answer value might either give the default + answer or ask you interactively, depending on whether retries are allowed (they by default are + allowed). So please test your scripts interactively before making them a non-interactive script. Directories: BORG_KEYS_DIR From 2f9b643edb37435eeed78c6516e93ef0fa143f64 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 06:34:09 +0100 Subject: [PATCH 291/321] migrate-to-repokey command, dispatch passphrase type to repokey handler every chunk has the encryption key type as first byte and we do not want to rewrite the whole repo to change the passphrase type to repokey type. thus we simply dispatch this type to repokey handler. if there is a repokey that contains the same secrets as they were derived from the passphrase, it will just work. if there is none yet, one needs to run migrate-to-repokey command to create it. --- borg/archiver.py | 45 ++++++++++++++++++++++++++++++++++++- borg/key.py | 12 ++++++---- borg/testsuite/archiver.py | 6 ++--- borg/testsuite/benchmark.py | 2 +- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5afb2f89..1503d152 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -26,7 +26,7 @@ from .compress import Compressor, COMPR_BUFFER from .upgrader import AtticRepositoryUpgrader from .repository import Repository from .cache import Cache -from .key import key_creator +from .key import key_creator, RepoKey, PassphraseKey from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS from .remote import RepositoryServer, RemoteRepository, cache_if_remote @@ -124,6 +124,23 @@ class Archiver: key.change_passphrase() return EXIT_SUCCESS + def do_migrate_to_repokey(self, args): + """Migrate passphrase -> repokey""" + repository = self.open_repository(args) + manifest_data = repository.get(Manifest.MANIFEST_ID) + key_old = PassphraseKey.detect(repository, manifest_data) + key_new = RepoKey(repository) + key_new.target = repository + key_new.repository_id = repository.id + key_new.enc_key = key_old.enc_key + key_new.enc_hmac_key = key_old.enc_hmac_key + key_new.id_key = key_old.id_key + key_new.chunk_seed = key_old.chunk_seed + key_new.change_passphrase() # option to change key protection passphrase, save + return EXIT_SUCCESS + + return EXIT_SUCCESS + def do_create(self, args): """Create new archive""" matcher = PatternMatcher(fallback=True) @@ -873,6 +890,32 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) + migrate_to_repokey_epilog = textwrap.dedent(""" + This command migrates a repository from passphrase mode (not supported any + more) to repokey mode. + + You will be first asked for the repository passphrase (to open it in passphrase + mode). This is the same passphrase as you used to use for this repo before 1.0. + + It will then derive the different secrets from this passphrase. + + Then you will be asked for a new passphrase (twice, for safety). This + passphrase will be used to protect the repokey (which contains these same + secrets in encrypted form). You may use the same passphrase as you used to + use, but you may also use a different one. + + After migrating to repokey mode, you can change the passphrase at any time. + But please note: the secrets will always stay the same and they could always + be derived from your (old) passphrase-mode passphrase. + """) + subparser = subparsers.add_parser('migrate-to-repokey', parents=[common_parser], + description=self.do_migrate_to_repokey.__doc__, + epilog=migrate_to_repokey_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_migrate_to_repokey) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False)) + create_epilog = textwrap.dedent(""" This command creates a backup archive containing all files found while recursively traversing all paths specified. The archive will consume almost no disk space for diff --git a/borg/key.py b/borg/key.py index 70999fb2..35058693 100644 --- a/borg/key.py +++ b/borg/key.py @@ -47,8 +47,10 @@ def key_factory(repository, manifest_data): return KeyfileKey.detect(repository, manifest_data) elif key_type == RepoKey.TYPE: return RepoKey.detect(repository, manifest_data) - elif key_type == PassphraseKey.TYPE: # deprecated, kill in 1.x - return PassphraseKey.detect(repository, manifest_data) + elif key_type == PassphraseKey.TYPE: + # this mode was killed in borg 1.0, see: https://github.com/borgbackup/borg/issues/97 + # we just dispatch to repokey mode and assume the passphrase was migrated to a repokey. + return RepoKey.detect(repository, manifest_data) elif key_type == PlaintextKey.TYPE: return PlaintextKey.detect(repository, manifest_data) else: @@ -132,7 +134,8 @@ class AESKeyBase(KeyBase): return b''.join((self.TYPE_STR, hmac, data)) def decrypt(self, id, data): - if data[0] != self.TYPE: + if not (data[0] == self.TYPE or \ + data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)): raise IntegrityError('Invalid encryption envelope') hmac_given = memoryview(data)[1:33] hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) @@ -148,7 +151,8 @@ class AESKeyBase(KeyBase): return data def extract_nonce(self, payload): - if payload[0] != self.TYPE: + if not (payload[0] == self.TYPE or \ + payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)): raise IntegrityError('Invalid encryption envelope') nonce = bytes_to_long(payload[33:41]) return nonce diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index ea9d1a72..95532bca 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -426,7 +426,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_repository_swap_detection(self): self.create_test_files() os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd('init', '--encryption=passphrase', self.repository_location) + self.cmd('init', '--encryption=repokey', self.repository_location) repository_id = self._extract_repository_id(self.repository_path) self.cmd('create', self.repository_location + '::test', 'input') shutil.rmtree(self.repository_path) @@ -442,7 +442,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_test_files() self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted') os.environ['BORG_PASSPHRASE'] = 'passphrase' - self.cmd('init', '--encryption=passphrase', self.repository_location + '_encrypted') + self.cmd('init', '--encryption=repokey', self.repository_location + '_encrypted') self.cmd('create', self.repository_location + '_encrypted::test', 'input') shutil.rmtree(self.repository_path + '_encrypted') os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') @@ -986,7 +986,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.verify_aes_counter_uniqueness('keyfile') def test_aes_counter_uniqueness_passphrase(self): - self.verify_aes_counter_uniqueness('passphrase') + self.verify_aes_counter_uniqueness('repokey') def test_debug_dump_archive_items(self): self.create_test_files() diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py index cb7f9e90..447a158e 100644 --- a/borg/testsuite/benchmark.py +++ b/borg/testsuite/benchmark.py @@ -25,7 +25,7 @@ def repo_url(request, tmpdir): tmpdir.remove(rec=1) -@pytest.fixture(params=["none", "passphrase"]) +@pytest.fixture(params=["none", "repokey"]) def repo(request, cmd, repo_url): cmd('init', '--encryption', request.param, repo_url) return repo_url From 815d2e23ce3358f934e91925439c60d4815679aa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 06:58:41 +0100 Subject: [PATCH 292/321] remove support for --encryption=passphrase, clean up --- borg/archiver.py | 4 +--- borg/key.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 1503d152..4a1a48cf 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -802,8 +802,6 @@ class Archiver: This command initializes an empty repository. A repository is a filesystem directory containing the deduplicated data from zero or more archives. Encryption can be enabled at repository init time. - Please note that the 'passphrase' encryption mode is DEPRECATED (instead of it, - consider using 'repokey'). """) subparser = subparsers.add_parser('init', parents=[common_parser], description=self.do_init.__doc__, epilog=init_epilog, @@ -813,7 +811,7 @@ class Archiver: type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', - choices=('none', 'keyfile', 'repokey', 'passphrase'), default='repokey', + choices=('none', 'keyfile', 'repokey'), default='repokey', help='select encryption key mode (default: "%(default)s")') check_epilog = textwrap.dedent(""" diff --git a/borg/key.py b/borg/key.py index 35058693..4cf4a955 100644 --- a/borg/key.py +++ b/borg/key.py @@ -35,8 +35,6 @@ def key_creator(repository, args): return KeyfileKey.create(repository, args) elif args.encryption == 'repokey': return RepoKey.create(repository, args) - elif args.encryption == 'passphrase': # deprecated, kill in 1.x - return PassphraseKey.create(repository, args) else: return PlaintextKey.create(repository, args) @@ -48,8 +46,8 @@ def key_factory(repository, manifest_data): elif key_type == RepoKey.TYPE: return RepoKey.detect(repository, manifest_data) elif key_type == PassphraseKey.TYPE: - # this mode was killed in borg 1.0, see: https://github.com/borgbackup/borg/issues/97 # we just dispatch to repokey mode and assume the passphrase was migrated to a repokey. + # see also comment in PassphraseKey class. return RepoKey.detect(repository, manifest_data) elif key_type == PlaintextKey.TYPE: return PlaintextKey.detect(repository, manifest_data) @@ -207,18 +205,21 @@ class Passphrase(str): class PassphraseKey(AESKeyBase): - # This mode is DEPRECATED and will be killed at 1.0 release. - # With this mode: + # This mode was killed in borg 1.0, see: https://github.com/borgbackup/borg/issues/97 + # Reasons: # - you can never ever change your passphrase for existing repos. # - you can never ever use a different iterations count for existing repos. + # "Killed" means: + # - there is no automatic dispatch to this class via type byte + # - --encryption=passphrase is an invalid argument now + # This class is kept for a while to support migration from passphrase to repokey mode. TYPE = 0x01 iterations = 100000 # must not be changed ever! @classmethod def create(cls, repository, args): key = cls(repository) - logger.warning('WARNING: "passphrase" mode is deprecated and will be removed in 1.0.') - logger.warning('If you want something similar (but with less issues), use "repokey" mode.') + logger.warning('WARNING: "passphrase" mode is unsupported since borg 1.0.') passphrase = Passphrase.new(allow_empty=False) key.init(repository, passphrase) return key From 1fc99ec9cd4a39cbb24338a67a219179ccfe7487 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 07:25:43 +0100 Subject: [PATCH 293/321] update docs, remove references to passphrase mode --- docs/internals.rst | 7 ++++++- docs/usage.rst | 13 +++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/internals.rst b/docs/internals.rst index 21ae2445..3dbf5da1 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -344,7 +344,12 @@ To reduce payload size, only 8 bytes of the 16 bytes nonce is saved in the payload, the first 8 bytes are always zeros. This does not affect security but limits the maximum repository capacity to only 295 exabytes (2**64 * 16 bytes). -Encryption keys are either derived from a passphrase or kept in a key file. +Encryption keys (and other secrets) are kept either in a key file on the client +('keyfile' mode) or in the repository config on the server ('repokey' mode). +In both cases, the secrets are generated from random and then encrypted by a +key derived from your passphrase (this happens on the client before the key +is stored into the keyfile or as repokey). + The passphrase is passed through the ``BORG_PASSPHRASE`` environment variable or prompted for interactive usage. diff --git a/docs/usage.rst b/docs/usage.rst index 3094f403..c9683648 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -198,12 +198,7 @@ an attacker has access to your backup repository. But be careful with the key / the passphrase: -``--encryption=passphrase`` is DEPRECATED and will be removed in next major release. -This mode has very fundamental, unfixable problems (like you can never change -your passphrase or the pbkdf2 iteration count for an existing repository, because -the encryption / decryption key is directly derived from the passphrase). - -If you want "passphrase-only" security, just use the ``repokey`` mode. The key will +If you want "passphrase-only" security, use the ``repokey`` mode. The key will be stored inside the repository (in its "config" file). In above mentioned attack scenario, the attacker will have the key (but not the passphrase). @@ -220,8 +215,10 @@ The backup that is encrypted with that key won't help you with that, of course. Make sure you use a good passphrase. Not too short, not too simple. The real encryption / decryption key is encrypted with / locked by your passphrase. If an attacker gets your key, he can't unlock and use it without knowing the -passphrase. In ``repokey`` and ``keyfile`` modes, you can change your passphrase -for existing repos. +passphrase. + +You can change your passphrase for existing repos at any time, it won't affect +the encryption/decryption key or other secrets. .. include:: usage/create.rst.inc From e1515ee251543355f22098cbcdfea9651ceae71b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 23:51:06 +0100 Subject: [PATCH 294/321] remove deprecated "borg verify" use borg extract --dry-run ... --- borg/archiver.py | 3 --- borg/testsuite/archiver.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 4a1a48cf..23fc0590 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -755,9 +755,6 @@ class Archiver: ('--do-not-cross-mountpoints', '--one-file-system', 'Warning: "--do-no-cross-mountpoints" has been deprecated. Use "--one-file-system" instead.'), ] - if args and args[0] == 'verify': - print('Warning: "borg verify" has been deprecated. Use "borg extract --dry-run" instead.') - args = ['extract', '--dry-run'] + args[1:] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: if arg.startswith(old_name): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 95532bca..c45db664 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -826,8 +826,6 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') - output = self.cmd('verify', '-v', self.repository_location + '::test') - self.assert_in('"borg verify" has been deprecated', output) output = self.cmd('prune', self.repository_location, '--hourly=1') self.assert_in('"--hourly" has been deprecated. Use "--keep-hourly" instead', output) From 079646ee4cabeb74c797ce74b688e6a6f680ff39 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 Jan 2016 23:53:57 +0100 Subject: [PATCH 295/321] remove deprecated "--do-not-cross-mountpoints" use --one-file-system instead --- borg/archiver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 23fc0590..fdfe5839 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -752,8 +752,6 @@ class Archiver: ('--weekly', '--keep-weekly', 'Warning: "--weekly" has been deprecated. Use "--keep-weekly" instead.'), ('--monthly', '--keep-monthly', 'Warning: "--monthly" has been deprecated. Use "--keep-monthly" instead.'), ('--yearly', '--keep-yearly', 'Warning: "--yearly" has been deprecated. Use "--keep-yearly" instead.'), - ('--do-not-cross-mountpoints', '--one-file-system', - 'Warning: "--do-no-cross-mountpoints" has been deprecated. Use "--one-file-system" instead.'), ] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: From ad31fcd7c08134fb7af852cdc1c560fa76dab9c1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 00:02:59 +0100 Subject: [PATCH 296/321] remove deprecated "--hourly/daily/weekly/monthly/yearly" use --keep-hourly/daily/weekly/monthly/yearly instead note: kept the code and test, we might have deprecated option in future, too --- borg/archiver.py | 6 +----- borg/testsuite/archiver.py | 12 ++++++------ 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index fdfe5839..48cae3e9 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -747,11 +747,7 @@ class Archiver: def preprocess_args(self, args): deprecations = [ - ('--hourly', '--keep-hourly', 'Warning: "--hourly" has been deprecated. Use "--keep-hourly" instead.'), - ('--daily', '--keep-daily', 'Warning: "--daily" has been deprecated. Use "--keep-daily" instead.'), - ('--weekly', '--keep-weekly', 'Warning: "--weekly" has been deprecated. Use "--keep-weekly" instead.'), - ('--monthly', '--keep-monthly', 'Warning: "--monthly" has been deprecated. Use "--keep-monthly" instead.'), - ('--yearly', '--keep-yearly', 'Warning: "--yearly" has been deprecated. Use "--keep-yearly" instead.'), + #('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), ] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index c45db664..bdea214f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -822,12 +822,12 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', '-v', '--list', '--filter=AM', self.repository_location + '::test3', 'input') self.assert_in('file1', output) - def test_cmdline_compatibility(self): - self.create_regular_file('file1', size=1024 * 80) - self.cmd('init', self.repository_location) - self.cmd('create', self.repository_location + '::test', 'input') - output = self.cmd('prune', self.repository_location, '--hourly=1') - self.assert_in('"--hourly" has been deprecated. Use "--keep-hourly" instead', output) + #def test_cmdline_compatibility(self): + # self.create_regular_file('file1', size=1024 * 80) + # self.cmd('init', self.repository_location) + # self.cmd('create', self.repository_location + '::test', 'input') + # output = self.cmd('foo', self.repository_location, '--old') + # self.assert_in('"--old" has been deprecated. Use "--new" instead', output) def test_prune_repository(self): self.cmd('init', self.repository_location) From 3476fffe7d4ceffafd8e50dc0ce56379cc06cfc1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 Jan 2016 00:12:11 +0100 Subject: [PATCH 297/321] remove deprecated "--compression " use --compression zlib, instead in case of 0, you could also use --compression none --- borg/helpers.py | 42 +++++++++++++-------------------------- borg/testsuite/helpers.py | 5 ----- 2 files changed, 14 insertions(+), 33 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 7957ee38..62694e0a 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -466,35 +466,21 @@ def CompressionSpec(s): count = len(values) if count < 1: raise ValueError - compression = values[0] - try: - compression = int(compression) - if count > 1: - raise ValueError - # DEPRECATED: it is just --compression N - if 0 <= compression <= 9: - print('Warning: --compression %d is deprecated, please use --compression zlib,%d.' % (compression, compression)) - if compression == 0: - print('Hint: instead of --compression zlib,0 you could also use --compression none for better performance.') - print('Hint: archives generated using --compression none are not compatible with borg < 0.25.0.') - return dict(name='zlib', level=compression) - raise ValueError - except ValueError: - # --compression algo[,...] - name = compression - if name in ('none', 'lz4', ): - return dict(name=name) - if name in ('zlib', 'lzma', ): - if count < 2: - level = 6 # default compression level in py stdlib - elif count == 2: - level = int(values[1]) - if not 0 <= level <= 9: - raise ValueError - else: + # --compression algo[,level] + name = values[0] + if name in ('none', 'lz4', ): + return dict(name=name) + if name in ('zlib', 'lzma', ): + if count < 2: + level = 6 # default compression level in py stdlib + elif count == 2: + level = int(values[1]) + if not 0 <= level <= 9: raise ValueError - return dict(name=name, level=level) - raise ValueError + else: + raise ValueError + return dict(name=name, level=level) + raise ValueError def dir_is_cachedir(path): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 82b3f135..53b2ed5b 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -458,11 +458,6 @@ def test_pattern_matcher(): def test_compression_specs(): with pytest.raises(ValueError): CompressionSpec('') - assert CompressionSpec('0') == dict(name='zlib', level=0) - assert CompressionSpec('1') == dict(name='zlib', level=1) - assert CompressionSpec('9') == dict(name='zlib', level=9) - with pytest.raises(ValueError): - CompressionSpec('10') assert CompressionSpec('none') == dict(name='none') assert CompressionSpec('lz4') == dict(name='lz4') assert CompressionSpec('zlib') == dict(name='zlib', level=6) From d2bfa248144127759e4580382a98803de5372caf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 24 Jan 2016 17:54:36 +0100 Subject: [PATCH 298/321] update CHANGES --- docs/changes.rst | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index de669bc3..ccbb744d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,85 @@ Changelog ========= +Version 1.0.0 (not released yet) +-------------------------------- + +The major release number change (0.x -> 1.x) indicates bigger incompatible +changes, please read the compatibility notes carefully and adapt / test your +scripts and carefully check your backup logs. + +Compatibility notes: + +- drop support for python 3.2 and 3.3, require 3.4 or 3.5, #221 #65 #490 + note: we provide binaries that include python 3.5.1 and everything else + needed. they are an option in case you are stuck with < 3.4 otherwise. +- change encryption to be on by default (using "repokey" mode) +- remove support for --encryption=passphrase, + use borg migrate-to-repokey to switch to repokey mode, #97 +- remove deprecated "--compression ", + use --compression zlib, instead + in case of 0, you could also use --compression none +- remove deprecated "--hourly/daily/weekly/monthly/yearly" + use --keep-hourly/daily/weekly/monthly/yearly instead +- remove deprecated "--do-not-cross-mountpoints", + use --one-file-system instead +- disambiguate -p option, #563: + + - -p now is same as --progress + - -P now is same as --prefix +- remove deprecated "borg verify", + use borg extract --dry-run instead +- cleanup environment variable semantics, #355 + the environment variables used to be "yes sayers" when set, this was + conceptually generalized to "automatic answerers" and they just give their + value as answer (as if you typed in that value when being asked). + See the "usage" / "Environment Variables" section of the docs for details. +- change the builtin default for --chunker-params, create 2MiB chunks, #343 + --chunker-params new default: 19,23,21,4095 - old default: 10,23,16,4096 + + one of the biggest issues with borg < 1.0 (and also attic) was that it had a + default target chunk size of 64kiB, thus it created a lot of chunks and thus + also a huge chunk management overhead (high RAM and disk usage). + + please note that the new default won't change the chunks that you already + have in your repository. the new big chunks do not deduplicate with the old + small chunks, so expect your repo to grow at least by the size of every + changed file and in the worst case (e.g. if your files cache was lost / is + not used) by the size of every file (minus any compression you might use). + + in case you want to immediately see a much lower resource usage (RAM / disk) + for chunks management, it might be better to start with a new repo than + continuing in the existing repo (with an existing repo, you'ld have to wait + until all archives with small chunks got pruned to see a lower resource + usage). + + if you used the old --chunker-params default value (or if you did not use + --chunker-params option at all) and you'ld like to continue using small + chunks (and you accept the huge resource usage that comes with that), just + explicitly use borg create --chunker-params=10,23,16,4095. + +New features: + +- borg migrate-to-repokey ("passphrase" -> "repokey" encryption key mode) + +Other changes: + +- suppress unneeded exception context (PEP 409), simpler tracebacks +- removed special code needed to deal with imperfections / incompatibilities / + missing stuff in py 3.2/3.3, simplify code that can be done simpler in 3.4 +- removed some version requirements that were kept on old versions because + newer did not support py 3.2 any more +- use some py 3.4+ stdlib code instead of own/openssl/pypi code: + + - use os.urandom instead of own cython openssl RAND_bytes wrapper, fixes #493 + - use hashlib.pbkdf2_hmac from py stdlib instead of own openssl wrapper + - use hmac.compare_digest instead of == operator (constant time comparison) + - use stat.filemode instead of homegrown code + - use "mock" library from stdlib, #145 + - remove borg.support (with non-broken argparse copy), it is ok in 3.4+, #358 +- Vagrant: copy CHANGES.rst as symlink, #592 + + Version 0.30.0 -------------- From 7cb6b5657a91efc3252230a1005706d832f67988 Mon Sep 17 00:00:00 2001 From: Herover Date: Sun, 24 Jan 2016 19:26:05 +0100 Subject: [PATCH 299/321] Fix dead link to license --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 4ac44031..9b8ff20d 100644 --- a/README.rst +++ b/README.rst @@ -112,7 +112,7 @@ Links * `Issue Tracker `_ * `Bounties & Fundraisers `_ * `Mailing List `_ -* `License `_ +* `License `_ Notes ----- From 37eb22ad5500545d8f3f13fc8158b711e95448f7 Mon Sep 17 00:00:00 2001 From: Gianfranco Costamagna Date: Mon, 25 Jan 2016 09:07:30 +0100 Subject: [PATCH 300/321] Delete Ubuntu Vivid, EOL Ubuntu Vivid is not supported anymore. --- docs/installation.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index f4a963dd..2d37d1a0 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -38,7 +38,6 @@ NetBSD `pkgsrc`_ ``pkg_add py-borgback NixOS `.nix file`_ N/A OS X `Brew cask`_ ``brew cask install borgbackup`` Ubuntu `Xenial 16.04`_, `Wily 15.10 (backport PPA)`_ ``apt install borgbackup`` -Ubuntu `Vivid 15.04 (backport PPA)`_ ``apt install borgbackup`` Ubuntu `Trusty 14.04 (backport PPA)`_ ``apt install borgbackup`` ============ ============================================= ======= @@ -48,7 +47,6 @@ Ubuntu `Trusty 14.04 (backport PPA)`_ ``apt install borgbac .. _pkgsrc: http://pkgsrc.se/sysutils/py-borgbackup .. _Xenial 16.04: https://launchpad.net/ubuntu/xenial/+source/borgbackup .. _Wily 15.10 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup -.. _Vivid 15.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup .. _Trusty 14.04 (backport PPA): https://launchpad.net/~costamagnagianfranco/+archive/ubuntu/borgbackup .. _.nix file: https://github.com/NixOS/nixpkgs/blob/master/pkgs/tools/backup/borg/default.nix .. _Brew cask: http://caskroom.io/ From 29b84a1b43e6c0c4fd53279ce2a3ba978db50793 Mon Sep 17 00:00:00 2001 From: Herover Date: Mon, 25 Jan 2016 13:33:48 +0100 Subject: [PATCH 301/321] Use https on updated license link --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9b8ff20d..0d2d800b 100644 --- a/README.rst +++ b/README.rst @@ -112,7 +112,7 @@ Links * `Issue Tracker `_ * `Bounties & Fundraisers `_ * `Mailing List `_ -* `License `_ +* `License `_ Notes ----- From 3061b3048c7cf44ab05a0a058eea235934195c27 Mon Sep 17 00:00:00 2001 From: Christoph Trassl Date: Mon, 25 Jan 2016 17:40:52 +0100 Subject: [PATCH 302/321] Reformat commands list. --- borg/archiver.py | 60 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 48cae3e9..a87da416 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -778,14 +778,16 @@ class Archiver: parser = argparse.ArgumentParser(prog=prog, description='Borg - Deduplicated Backups') parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, help='show version number and exit') - subparsers = parser.add_subparsers(title='Available commands') + subparsers = parser.add_subparsers(title='required arguments', + metavar='') serve_epilog = textwrap.dedent(""" This command starts a repository server process. This command is usually not used manually. """) subparser = subparsers.add_parser('serve', parents=[common_parser], description=self.do_serve.__doc__, epilog=serve_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='start repository server process') subparser.set_defaults(func=self.do_serve) subparser.add_argument('--restrict-to-path', dest='restrict_to_paths', action='append', metavar='PATH', help='restrict repository access to PATH') @@ -796,7 +798,8 @@ class Archiver: """) subparser = subparsers.add_parser('init', parents=[common_parser], description=self.do_init.__doc__, epilog=init_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='initialize empty repository') subparser.set_defaults(func=self.do_init) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), @@ -844,7 +847,8 @@ class Archiver: subparser = subparsers.add_parser('check', parents=[common_parser], description=self.do_check.__doc__, epilog=check_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='verify repository') subparser.set_defaults(func=self.do_check) subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), @@ -874,7 +878,8 @@ class Archiver: subparser = subparsers.add_parser('change-passphrase', parents=[common_parser], description=self.do_change_passphrase.__doc__, epilog=change_passphrase_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='change repository passphrase') subparser.set_defaults(func=self.do_change_passphrase) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) @@ -900,7 +905,8 @@ class Archiver: subparser = subparsers.add_parser('migrate-to-repokey', parents=[common_parser], description=self.do_migrate_to_repokey.__doc__, epilog=migrate_to_repokey_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='migrate passphrase-mode repository to repokey') subparser.set_defaults(func=self.do_migrate_to_repokey) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) @@ -916,7 +922,8 @@ class Archiver: subparser = subparsers.add_parser('create', parents=[common_parser], description=self.do_create.__doc__, epilog=create_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='create backup') subparser.set_defaults(func=self.do_create) subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, @@ -996,7 +1003,8 @@ class Archiver: subparser = subparsers.add_parser('extract', parents=[common_parser], description=self.do_extract.__doc__, epilog=extract_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='extract archive contents') subparser.set_defaults(func=self.do_extract) subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', @@ -1031,7 +1039,8 @@ class Archiver: subparser = subparsers.add_parser('rename', parents=[common_parser], description=self.do_rename.__doc__, epilog=rename_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='rename archive') subparser.set_defaults(func=self.do_rename) subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), @@ -1047,7 +1056,8 @@ class Archiver: subparser = subparsers.add_parser('delete', parents=[common_parser], description=self.do_delete.__doc__, epilog=delete_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='delete archive') subparser.set_defaults(func=self.do_delete) subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, @@ -1071,7 +1081,8 @@ class Archiver: subparser = subparsers.add_parser('list', parents=[common_parser], description=self.do_list.__doc__, epilog=list_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='list archive or repository contents') subparser.set_defaults(func=self.do_list) subparser.add_argument('--short', dest='short', action='store_true', default=False, @@ -1091,7 +1102,8 @@ class Archiver: subparser = subparsers.add_parser('mount', parents=[common_parser], description=self.do_mount.__doc__, epilog=mount_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='mount repository') subparser.set_defaults(func=self.do_mount) subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', type=location_validator(), help='repository/archive to mount') @@ -1109,7 +1121,8 @@ class Archiver: subparser = subparsers.add_parser('info', parents=[common_parser], description=self.do_info.__doc__, epilog=info_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='show archive information') subparser.set_defaults(func=self.do_info) subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), @@ -1123,7 +1136,8 @@ class Archiver: subparser = subparsers.add_parser('break-lock', parents=[common_parser], description=self.do_break_lock.__doc__, epilog=break_lock_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='break repository and cache locks') subparser.set_defaults(func=self.do_break_lock) subparser.add_argument('location', metavar='REPOSITORY', type=location_validator(archive=False), @@ -1156,7 +1170,8 @@ class Archiver: subparser = subparsers.add_parser('prune', parents=[common_parser], description=self.do_prune.__doc__, epilog=prune_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='prune archives') subparser.set_defaults(func=self.do_prune) subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', @@ -1224,7 +1239,8 @@ class Archiver: subparser = subparsers.add_parser('upgrade', parents=[common_parser], description=self.do_upgrade.__doc__, epilog=upgrade_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='upgrade repository format') subparser.set_defaults(func=self.do_upgrade) subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, @@ -1256,7 +1272,8 @@ class Archiver: subparser = subparsers.add_parser('debug-dump-archive-items', parents=[common_parser], description=self.do_debug_dump_archive_items.__doc__, epilog=debug_dump_archive_items_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='dump archive items (metadata) (debug)') subparser.set_defaults(func=self.do_debug_dump_archive_items) subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), @@ -1268,7 +1285,8 @@ class Archiver: subparser = subparsers.add_parser('debug-get-obj', parents=[common_parser], description=self.do_debug_get_obj.__doc__, epilog=debug_get_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='get object from repository (debug)') subparser.set_defaults(func=self.do_debug_get_obj) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), @@ -1284,7 +1302,8 @@ class Archiver: subparser = subparsers.add_parser('debug-put-obj', parents=[common_parser], description=self.do_debug_put_obj.__doc__, epilog=debug_put_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='put object to repository (debug)') subparser.set_defaults(func=self.do_debug_put_obj) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), @@ -1298,7 +1317,8 @@ class Archiver: subparser = subparsers.add_parser('debug-delete-obj', parents=[common_parser], description=self.do_debug_delete_obj.__doc__, epilog=debug_delete_obj_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter) + formatter_class=argparse.RawDescriptionHelpFormatter, + help='delete object from repository (debug)') subparser.set_defaults(func=self.do_debug_delete_obj) subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False), From dcffa5c6a243a535e5acd63d621ec53224af4f31 Mon Sep 17 00:00:00 2001 From: Adam Kouse Date: Tue, 26 Jan 2016 11:51:38 -0500 Subject: [PATCH 303/321] Single quote exclude line that includes an asterisk to prevent shell expansion --- docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 57e5481f..42d19618 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -105,7 +105,7 @@ certain number of old archives:: $REPOSITORY::`hostname`-`date +%Y-%m-%d` \ /home \ /var/www \ - --exclude /home/*/.cache \ + --exclude '/home/*/.cache' \ --exclude /home/Ben/Music/Justin\ Bieber \ --exclude '*.pyc' From e7c2189a3f591cc3ca572eb2fdcc42c8e36ac00a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Jan 2016 20:25:55 +0100 Subject: [PATCH 304/321] implement --list for borg extract before, borg extract always emitted the full file list at info log level. now, you need to give --list to get the full file list (consistent with borg create --list). this is currently only useful if you also use other output-generating options, e.g. --show-rc - you can now leave away --list to only get that other output. later, if extract gets more output-generating options, --list will get more useful. --- borg/archiver.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index a87da416..96993757 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -318,6 +318,7 @@ class Archiver: matcher.fallback = not include_patterns + output_list = args.output_list dry_run = args.dry_run stdout = args.stdout sparse = args.sparse @@ -332,7 +333,8 @@ class Archiver: if not args.dry_run: while dirs and not item[b'path'].startswith(dirs[-1][b'path']): archive.extract_item(dirs.pop(-1), stdout=stdout) - logger.info(remove_surrogates(orig_path)) + if output_list: + logger.info(remove_surrogates(orig_path)) try: if dry_run: archive.extract_item(item, dry_run=True) @@ -1006,6 +1008,9 @@ class Archiver: formatter_class=argparse.RawDescriptionHelpFormatter, help='extract archive contents') subparser.set_defaults(func=self.do_extract) + subparser.add_argument('--list', dest='output_list', + action='store_true', default=False, + help='output verbose list of items (files, dirs, ...)') subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not actually change any files') From 824e548b9fb1249af0df90331733a09ec240cc29 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Jan 2016 20:32:30 +0100 Subject: [PATCH 305/321] add missing example for --list option of borg create --- docs/usage.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index c9683648..b025af41 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -230,6 +230,9 @@ Examples # Backup ~/Documents into an archive named "my-documents" $ borg create /mnt/backup::my-documents ~/Documents + # same, but verbosely list all files as we process them + $ borg create -v --list /mnt/backup::my-documents ~/Documents + # Backup ~/Documents and ~/src but exclude pyc files $ borg create /mnt/backup::my-files \ ~/Documents \ From 7ea2404048eeb8d5cbe45ec31a5c5e9594f8925c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Jan 2016 21:59:24 +0100 Subject: [PATCH 306/321] borg serve: overwrite client's --restrict-to-path with forced command's option value, fixes #544 we also make sure the client is not cheating, like giving another subcommand or his own --restrict-to-path. --- borg/archiver.py | 17 ++++++++++++++++- borg/testsuite/archiver.py | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 96993757..52cca198 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -7,6 +7,7 @@ import functools import inspect import io import os +import shlex import signal import stat import sys @@ -1332,6 +1333,20 @@ class Archiver: help='hex object ID(s) to delete from the repo') return parser + def get_args(self, argv, cmd): + """usually, just returns argv, except if we deal with a ssh forced command for borg serve.""" + result = self.parse_args(argv[1:]) + if cmd is not None and result.func == self.do_serve: + forced_result = result + argv = shlex.split(cmd) + result = self.parse_args(argv[1:]) + if result.func != forced_result.func: + # someone is trying to execute a different borg subcommand, don't do that! + return forced_result + # the only thing we take from the forced "borg serve" ssh command is --restrict-to-path + result.restrict_to_paths = forced_result.restrict_to_paths + return result + def parse_args(self, args=None): # We can't use argparse for "serve" since we don't want it to show up in "Available commands" if args: @@ -1407,7 +1422,7 @@ def main(): # pragma: no cover setup_signal_handlers() archiver = Archiver() msg = None - args = archiver.parse_args(sys.argv[1:]) + args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) try: exit_code = archiver.run(args) except Error as e: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index bdea214f..d6cf8fbf 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1129,3 +1129,24 @@ class RemoteArchiverTestCase(ArchiverTestCase): @unittest.skip('only works locally') def test_debug_put_get_delete_obj(self): pass + + +def test_get_args(): + archiver = Archiver() + # everything normal: + # first param is argv as produced by ssh forced command, + # second param is like from SSH_ORIGINAL_COMMAND env variable + args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], + 'borg serve --info --umask=0027') + assert args.func == archiver.do_serve + assert args.restrict_to_paths == ['/p1', '/p2'] + assert args.umask == 0o027 + assert args.log_level == 'info' + # trying to cheat - break out of path restriction + args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], + 'borg serve --restrict-to-path=/') + assert args.restrict_to_paths == ['/p1', '/p2'] + # trying to cheat - try to execute different subcommand + args = archiver.get_args(['borg', 'serve', '--restrict-to-path=/p1', '--restrict-to-path=/p2', ], + 'borg init /') + assert args.func == archiver.do_serve From b8d954e60a50c8e432c657a3b2d0ce250213ebfb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Jan 2016 22:26:58 +0100 Subject: [PATCH 307/321] use XDG_CONFIG_HOME for borg keys instead of ~/.borg, fixes #515 --- borg/helpers.py | 4 ++-- borg/testsuite/helpers.py | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 62694e0a..28231a27 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -210,8 +210,8 @@ class Statistics: def get_keys_dir(): """Determine where to repository keys and cache""" - return os.environ.get('BORG_KEYS_DIR', - os.path.join(os.path.expanduser('~'), '.borg', 'keys')) + xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) + return os.environ.get('BORG_KEYS_DIR', os.path.join(xdg_config, 'borg', 'keys')) def get_cache_dir(): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 53b2ed5b..0d68fe97 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -10,7 +10,8 @@ import msgpack import msgpack.fallback from ..helpers import Location, format_file_size, format_timedelta, make_path_safe, \ - prune_within, prune_split, get_cache_dir, Statistics, is_slow_msgpack, yes, TRUISH, FALSISH, DEFAULTISH, \ + prune_within, prune_split, get_cache_dir, get_keys_dir, Statistics, is_slow_msgpack, \ + yes, TRUISH, FALSISH, DEFAULTISH, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, load_excludes, parse_pattern, \ PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern @@ -579,7 +580,7 @@ class TestParseTimestamp(BaseTestCase): def test_get_cache_dir(): - """test that get_cache_dir respects environement""" + """test that get_cache_dir respects environment""" # reset BORG_CACHE_DIR in order to test default old_env = None if os.environ.get('BORG_CACHE_DIR'): @@ -595,6 +596,23 @@ def test_get_cache_dir(): os.environ['BORG_CACHE_DIR'] = old_env +def test_get_keys_dir(): + """test that get_keys_dir respects environment""" + # reset BORG_KEYS_DIR in order to test default + old_env = None + if os.environ.get('BORG_KEYS_DIR'): + old_env = os.environ['BORG_KEYS_DIR'] + del(os.environ['BORG_KEYS_DIR']) + assert get_keys_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'keys') + os.environ['XDG_CONFIG_HOME'] = '/var/tmp/.config' + assert get_keys_dir() == os.path.join('/var/tmp/.config', 'borg', 'keys') + os.environ['BORG_KEYS_DIR'] = '/var/tmp' + assert get_keys_dir() == '/var/tmp' + # reset old env + if old_env is not None: + os.environ['BORG_KEYS_DIR'] = old_env + + @pytest.fixture() def stats(): stats = Statistics() From e06b7162c296254d81763cf370a4206f77f18353 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 28 Jan 2016 23:15:49 +0100 Subject: [PATCH 308/321] update docs / docstring about new key location --- borg/archiver.py | 2 +- borg/upgrader.py | 2 +- docs/internals.rst | 2 +- docs/quickstart.rst | 2 +- docs/usage.rst | 14 +++++++------- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 96993757..f704f536 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1213,7 +1213,7 @@ class Archiver: It will change the magic strings in the repository's segments to match the new Borg magic strings. The keyfiles found in $ATTIC_KEYS_DIR or ~/.attic/keys/ will also be converted and - copied to $BORG_KEYS_DIR or ~/.borg/keys. + copied to $BORG_KEYS_DIR or ~/.config/borg/keys. The cache files are converted, from $ATTIC_CACHE_DIR or ~/.cache/attic to $BORG_CACHE_DIR or ~/.cache/borg, but the diff --git a/borg/upgrader.py b/borg/upgrader.py index d836c57a..2d8856ee 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -137,7 +137,7 @@ class AtticRepositoryUpgrader(Repository): replacement pattern is `s/ATTIC KEY/BORG_KEY/` in `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or - `$HOME/.borg/keys`. + `$HOME/.config/borg/keys`. no need to decrypt to convert. we need to rewrite the whole key file because magic string length changed, but that's not a diff --git a/docs/internals.rst b/docs/internals.rst index 3dbf5da1..fbbef987 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -358,7 +358,7 @@ Key files --------- When initialized with the ``init -e keyfile`` command, |project_name| -needs an associated file in ``$HOME/.borg/keys`` to read and write +needs an associated file in ``$HOME/.config/borg/keys`` to read and write the repository. The format is based on msgpack_, base64 encoding and PBKDF2_ SHA256 hashing, which is then encoded again in a msgpack_. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 42d19618..4e0d7b5c 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -167,7 +167,7 @@ is being made. protection. The repository server never sees the plaintext key. ``keyfile`` mode - The key is stored on your local disk (in ``~/.borg/keys/``). + The key is stored on your local disk (in ``~/.config/borg/keys/``). Use this mode if you want "passphrase and having-the-key" security. In both modes, the key is stored in encrypted form and can be only decrypted diff --git a/docs/usage.rst b/docs/usage.rst index b025af41..22e783f3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -85,7 +85,7 @@ Some automatic "answerers" (if set, they automatically answer confirmation quest Directories: BORG_KEYS_DIR - Default to '~/.borg/keys'. This directory contains keys for encrypted repositories. + Default to '~/.config/borg/keys'. This directory contains keys for encrypted repositories. BORG_CACHE_DIR Default to '~/.cache/borg'. This directory contains the local cache and might need a lot of space for dealing with big repositories). @@ -203,9 +203,9 @@ be stored inside the repository (in its "config" file). In above mentioned attack scenario, the attacker will have the key (but not the passphrase). If you want "passphrase and having-the-key" security, use the ``keyfile`` mode. -The key will be stored in your home directory (in ``.borg/keys``). In the attack -scenario, the attacker who has just access to your repo won't have the key (and -also not the passphrase). +The key will be stored in your home directory (in ``.config/borg/keys``). In +the attack scenario, the attacker who has just access to your repo won't have +the key (and also not the passphrase). Make a backup copy of the key file (``keyfile`` mode) or repo config file (``repokey`` mode) and keep it at a safe place, so you still have the key in @@ -411,15 +411,15 @@ Examples Initializing repository at "/mnt/backup" Enter passphrase (empty for no passphrase): Enter same passphrase again: - Key file "/home/USER/.borg/keys/mnt_backup" created. + Key file "/home/USER/.config/borg/keys/mnt_backup" created. Keep this file safe. Your data will be inaccessible without it. # Change key file passphrase $ borg change-passphrase /mnt/backup - Enter passphrase for key file /home/USER/.borg/keys/mnt_backup: + Enter passphrase for key file /home/USER/.config/borg/keys/mnt_backup: New passphrase: Enter same passphrase again: - Key file "/home/USER/.borg/keys/mnt_backup" updated + Key file "/home/USER/.config/borg/keys/mnt_backup" updated .. include:: usage/serve.rst.inc From e7add135a22616d53c698567e2a4ce04b7442a4e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 29 Jan 2016 01:23:24 +0100 Subject: [PATCH 309/321] add upgrader which moves the keys to new location --- borg/archiver.py | 7 ++++++- borg/upgrader.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index f704f536..1e5e80ee 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -23,7 +23,7 @@ from .helpers import Error, location_validator, format_time, format_file_size, \ from .logger import create_logger, setup_logging logger = create_logger() from .compress import Compressor, COMPR_BUFFER -from .upgrader import AtticRepositoryUpgrader +from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader from .repository import Repository from .cache import Cache from .key import key_creator, RepoKey, PassphraseKey @@ -556,6 +556,11 @@ class Archiver: # XXX: should auto-detect if it is an attic repository here repo = AtticRepositoryUpgrader(args.location.path, create=False) + try: + repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress) + except NotImplementedError as e: + print("warning: %s" % e) + repo = BorgRepositoryUpgrader(args.location.path, create=False) try: repo.upgrade(args.dry_run, inplace=args.inplace, progress=args.progress) except NotImplementedError as e: diff --git a/borg/upgrader.py b/borg/upgrader.py index 2d8856ee..6884f7d3 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -276,3 +276,52 @@ class AtticKeyfileKey(KeyfileKey): if line and line.startswith(cls.FILE_ID) and line[10:] == id: return filename raise KeyfileNotFoundError(repository.path, keys_dir) + + +class BorgRepositoryUpgrader(Repository): + def upgrade(self, dryrun=True, inplace=False, progress=False): + """convert an old borg repository to a current borg repository + """ + logger.info("converting borg 0.xx to borg current") + try: + keyfile = self.find_borg0xx_keyfile() + except KeyfileNotFoundError: + logger.warning("no key file found for repository") + else: + self.move_keyfiles(keyfile, dryrun) + + def find_borg0xx_keyfile(self): + return Borg0xxKeyfileKey.find_key_file(self) + + def move_keyfiles(self, keyfile, dryrun): + filename = os.path.basename(keyfile) + new_keyfile = os.path.join(get_keys_dir(), filename) + try: + os.rename(keyfile, new_keyfile) + except FileExistsError: + # likely the attic -> borg upgrader already put it in the final location + pass + + +class Borg0xxKeyfileKey(KeyfileKey): + """backwards compatible borg 0.xx key file parser""" + + @staticmethod + def get_keys_dir(): + return os.environ.get('BORG_KEYS_DIR', + os.path.join(os.path.expanduser('~'), '.borg', 'keys')) + + @classmethod + def find_key_file(cls, repository): + get_keys_dir = cls.get_keys_dir + id = hexlify(repository.id).decode('ascii') + keys_dir = get_keys_dir() + if not os.path.exists(keys_dir): + raise KeyfileNotFoundError(repository.path, keys_dir) + for name in os.listdir(keys_dir): + filename = os.path.join(keys_dir, name) + with open(filename, 'r') as fd: + line = fd.readline().strip() + if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID)+1:] == id: + return filename + raise KeyfileNotFoundError(repository.path, keys_dir) From 695dc684794daa47152a8182b3855c0f08992d60 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 29 Jan 2016 01:40:29 +0100 Subject: [PATCH 310/321] slightly rephrase prune help --- borg/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 96993757..6df53482 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1153,8 +1153,8 @@ class Archiver: any of the specified retention options. This command is normally used by automated backup scripts wanting to keep a certain number of historic backups. - As an example, "-d 7" means to keep the latest backup on each day for 7 days. - Days without backups do not count towards the total. + As an example, "-d 7" means to keep the latest backup on each day, up to 7 + most recent days with backups (days without backups do not count). The rules are applied from hourly to yearly, and backups selected by previous rules do not count towards those of later rules. The time that each backup completes is used for pruning purposes. Dates and times are interpreted in From 12fe47fcd28e7166320b75e3be89333ad345e977 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 29 Jan 2016 01:50:51 +0100 Subject: [PATCH 311/321] implement --short for borg list REPO, fixes #611 --- borg/archiver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6df53482..331a3970 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -474,7 +474,10 @@ class Archiver: for archive_info in manifest.list_archive_infos(sort_by='ts'): if args.prefix and not archive_info.name.startswith(args.prefix): continue - print(format_archive(archive_info)) + if args.short: + print(archive_info.name) + else: + print(format_archive(archive_info)) return self.exit_code def do_info(self, args): From 7773e632dbb14a77ad4e6a866745445152edb8f9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 00:01:13 +0100 Subject: [PATCH 312/321] fix some minor cosmetic code/docs issues --- borg/archiver.py | 2 -- borg/helpers.py | 1 - borg/key.py | 4 ++-- borg/locking.py | 1 - borg/remote.py | 1 - borg/upgrader.py | 1 - docs/global.rst.inc | 1 - 7 files changed, 2 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6df53482..71165858 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -139,8 +139,6 @@ class Archiver: key_new.change_passphrase() # option to change key protection passphrase, save return EXIT_SUCCESS - return EXIT_SUCCESS - def do_create(self, args): """Create new archive""" matcher = PatternMatcher(fallback=True) diff --git a/borg/helpers.py b/borg/helpers.py index 62694e0a..7d592952 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -1,5 +1,4 @@ import argparse -import binascii from collections import namedtuple from functools import wraps import grp diff --git a/borg/key.py b/borg/key.py index 4cf4a955..82dbf3c7 100644 --- a/borg/key.py +++ b/borg/key.py @@ -132,7 +132,7 @@ class AESKeyBase(KeyBase): return b''.join((self.TYPE_STR, hmac, data)) def decrypt(self, id, data): - if not (data[0] == self.TYPE or \ + if not (data[0] == self.TYPE or data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)): raise IntegrityError('Invalid encryption envelope') hmac_given = memoryview(data)[1:33] @@ -149,7 +149,7 @@ class AESKeyBase(KeyBase): return data def extract_nonce(self, payload): - if not (payload[0] == self.TYPE or \ + if not (payload[0] == self.TYPE or payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)): raise IntegrityError('Invalid encryption envelope') nonce = bytes_to_long(payload[33:41]) diff --git a/borg/locking.py b/borg/locking.py index 99d20641..bf7ed603 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -1,4 +1,3 @@ -import errno import json import os import socket diff --git a/borg/remote.py b/borg/remote.py index a836ac2c..6ddc3f1c 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -7,7 +7,6 @@ import shlex from subprocess import Popen, PIPE import sys import tempfile -import traceback from . import __version__ diff --git a/borg/upgrader.py b/borg/upgrader.py index d836c57a..7f88fa60 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -4,7 +4,6 @@ import logging logger = logging.getLogger(__name__) import os import shutil -import sys import time from .helpers import get_keys_dir, get_cache_dir, ProgressIndicatorPercent diff --git a/docs/global.rst.inc b/docs/global.rst.inc index 439b71d1..d34f0965 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -23,7 +23,6 @@ .. _llfuse: https://pypi.python.org/pypi/llfuse/ .. _homebrew: http://brew.sh/ .. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace -.. _librelist: http://librelist.com/ .. _Cython: http://cython.org/ .. _virtualenv: https://pypi.python.org/pypi/virtualenv/ .. _mailing list discussion about internals: http://librelist.com/browser/attic/2014/5/6/questions-and-suggestions-about-inner-working-of-attic> From 8ec62d5e2e830de3fae3760d12c025ff6f8875ac Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 00:39:25 +0100 Subject: [PATCH 313/321] use os.path.normpath on repository paths, fixes #606 this does NOT fix absolute vs. relative path usage, but as this also deals with remote paths, this can't be done in general. --- borg/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 7d592952..3775e293 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -688,20 +688,20 @@ class Location: self.user = m.group('user') self.host = m.group('host') self.port = m.group('port') and int(m.group('port')) or None - self.path = m.group('path') + self.path = os.path.normpath(m.group('path')) self.archive = m.group('archive') return True m = self.file_re.match(text) if m: self.proto = m.group('proto') - self.path = m.group('path') + self.path = os.path.normpath(m.group('path')) self.archive = m.group('archive') return True m = self.scp_re.match(text) if m: self.user = m.group('user') self.host = m.group('host') - self.path = m.group('path') + self.path = os.path.normpath(m.group('path')) self.archive = m.group('archive') self.proto = self.host and 'ssh' or 'file' return True From 4b339f5d69eaea253b9ad7f7a45d757f44209851 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 21:32:45 +0100 Subject: [PATCH 314/321] cosmetic source cleanup (flake8) --- borg/__main__.py | 1 - borg/archive.py | 5 ++--- borg/archiver.py | 12 +++++------- borg/cache.py | 4 ++-- borg/fuse.py | 2 +- borg/helpers.py | 10 +++++----- borg/key.py | 2 +- borg/locking.py | 2 +- borg/remote.py | 2 +- borg/testsuite/archiver.py | 20 +++++++++++--------- borg/testsuite/benchmark.py | 9 +++++---- borg/testsuite/compress.py | 2 -- borg/testsuite/helpers.py | 5 ++--- borg/testsuite/locking.py | 1 + borg/testsuite/repository.py | 2 +- borg/upgrader.py | 2 +- borg/xattr.py | 4 ++-- setup.py | 5 ++++- 18 files changed, 45 insertions(+), 45 deletions(-) diff --git a/borg/__main__.py b/borg/__main__.py index b38dc4e9..3e7f4745 100644 --- a/borg/__main__.py +++ b/borg/__main__.py @@ -1,3 +1,2 @@ from borg.archiver import main main() - diff --git a/borg/archive.py b/borg/archive.py index ad22d3d1..ab7ff199 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -187,7 +187,7 @@ class Archive: @property def duration(self): - return format_timedelta(self.end-self.start) + return format_timedelta(self.end - self.start) def __str__(self): return '''Archive name: {0.name} @@ -591,8 +591,7 @@ Number of files: {0.stats.nfiles}'''.format(self) # this set must be kept complete, otherwise the RobustUnpacker might malfunction: ITEM_KEYS = set([b'path', b'source', b'rdev', b'chunks', b'mode', b'user', b'group', b'uid', b'gid', b'mtime', b'atime', b'ctime', - b'xattrs', b'bsdflags', b'acl_nfs4', b'acl_access', b'acl_default', b'acl_extended', - ]) + b'xattrs', b'bsdflags', b'acl_nfs4', b'acl_access', b'acl_default', b'acl_extended', ]) class RobustUnpacker: diff --git a/borg/archiver.py b/borg/archiver.py index eb79446e..115b2bc3 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -239,8 +239,7 @@ class Archiver: # Ignore if nodump flag is set if has_lchflags and (st.st_flags & stat.UF_NODUMP): return - if (stat.S_ISREG(st.st_mode) or - read_special and not stat.S_ISDIR(st.st_mode)): + if stat.S_ISREG(st.st_mode) or read_special and not stat.S_ISDIR(st.st_mode): if not dry_run: try: status = archive.process_file(path, st, cache) @@ -576,7 +575,7 @@ class Archiver: archive = Archive(repository, key, manifest, args.location.archive) for i, item_id in enumerate(archive.metadata[b'items']): data = key.decrypt(item_id, repository.get(item_id)) - filename = '%06d_%s.items' %(i, hexlify(item_id).decode('ascii')) + filename = '%06d_%s.items' % (i, hexlify(item_id).decode('ascii')) print('Dumping', filename) with open(filename, 'wb') as fd: fd.write(data) @@ -594,7 +593,7 @@ class Archiver: print("object id %s is invalid." % hex_id) else: try: - data =repository.get(id) + data = repository.get(id) except repository.ObjectNotFound: print("object %s not found." % hex_id) else: @@ -756,7 +755,7 @@ class Archiver: def preprocess_args(self, args): deprecations = [ - #('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), + # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), ] for i, arg in enumerate(args[:]): for old_name, new_name, warning in deprecations: @@ -787,8 +786,7 @@ class Archiver: parser = argparse.ArgumentParser(prog=prog, description='Borg - Deduplicated Backups') parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, help='show version number and exit') - subparsers = parser.add_subparsers(title='required arguments', - metavar='') + subparsers = parser.add_subparsers(title='required arguments', metavar='') serve_epilog = textwrap.dedent(""" This command starts a repository server process. This command is usually not used manually. diff --git a/borg/cache.py b/borg/cache.py index 521b6846..ea3ecb97 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -130,10 +130,10 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" try: cache_version = self.config.getint('cache', 'version') wanted_version = 1 - if cache_version != wanted_version: + if cache_version != wanted_version: raise Exception('%s has unexpected cache version %d (wanted: %d).' % ( config_path, cache_version, wanted_version)) - except configparser.NoSectionError as e: + except configparser.NoSectionError: raise Exception('%s does not look like a Borg cache.' % config_path) from None self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) diff --git a/borg/fuse.py b/borg/fuse.py index a7c8845b..c726a563 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -209,7 +209,7 @@ class FuseOperations(llfuse.Operations): continue n = min(size, s - offset) chunk = self.key.decrypt(id, self.repository.get(id)) - parts.append(chunk[offset:offset+n]) + parts.append(chunk[offset:offset + n]) offset = 0 size -= n if not size: diff --git a/borg/helpers.py b/borg/helpers.py index e42dfd77..4f230e6d 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -128,7 +128,7 @@ class Manifest: def prune_within(archives, within): - multiplier = {'H': 1, 'd': 24, 'w': 24*7, 'm': 24*31, 'y': 24*365} + multiplier = {'H': 1, 'd': 24, 'w': 24 * 7, 'm': 24 * 31, 'y': 24 * 365} try: hours = int(within[:-1]) * multiplier[within[-1]] except (KeyError, ValueError): @@ -136,7 +136,7 @@ def prune_within(archives, within): raise argparse.ArgumentTypeError('Unable to parse --within option: "%s"' % within) if hours <= 0: raise argparse.ArgumentTypeError('Number specified using --within option must be positive') - target = datetime.now(timezone.utc) - timedelta(seconds=hours*60*60) + target = datetime.now(timezone.utc) - timedelta(seconds=hours * 3600) return [a for a in archives if a.ts > target] @@ -200,7 +200,7 @@ class Statistics: path = remove_surrogates(item[b'path']) if item else '' space = columns - len(msg) if space < len('...') + len(path): - path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:]) + path = '%s...%s' % (path[:(space // 2) - len('...')], path[-space // 2:]) msg += "{0:<{space}}".format(path, space=space) else: msg = ' ' * columns @@ -355,7 +355,7 @@ class FnmatchPattern(PatternBase): if pattern.endswith(os.path.sep): pattern = os.path.normpath(pattern).rstrip(os.path.sep) + os.path.sep + '*' + os.path.sep else: - pattern = os.path.normpath(pattern) + os.path.sep+'*' + pattern = os.path.normpath(pattern) + os.path.sep + '*' self.pattern = pattern @@ -831,6 +831,7 @@ FALSISH = ('No', 'NO', 'no', 'N', 'n', '0', ) TRUISH = ('Yes', 'YES', 'yes', 'Y', 'y', '1', ) DEFAULTISH = ('Default', 'DEFAULT', 'default', 'D', 'd', '', ) + def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, retry_msg=None, invalid_msg=None, env_msg=None, falsish=FALSISH, truish=TRUISH, defaultish=DEFAULTISH, @@ -951,7 +952,6 @@ class ProgressIndicatorPercent: print(" " * len(self.msg % 100.0), file=self.file, end='\r') - class ProgressIndicatorEndless: def __init__(self, step=10, file=sys.stderr): """ diff --git a/borg/key.py b/borg/key.py index 82dbf3c7..6e56de19 100644 --- a/borg/key.py +++ b/borg/key.py @@ -360,7 +360,7 @@ class KeyfileKey(KeyfileKeyBase): filename = os.path.join(keys_dir, name) with open(filename, 'r') as fd: line = fd.readline().strip() - if line.startswith(self.FILE_ID) and line[len(self.FILE_ID)+1:] == id: + if line.startswith(self.FILE_ID) and line[len(self.FILE_ID) + 1:] == id: return filename raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir()) diff --git a/borg/locking.py b/borg/locking.py index bf7ed603..c0388578 100644 --- a/borg/locking.py +++ b/borg/locking.py @@ -111,7 +111,7 @@ class ExclusiveLock: self.sleep = sleep self.path = os.path.abspath(path) self.id = id or get_id() - self.unique_name = os.path.join(self.path, "%s.%d-%x" % self.id) + self.unique_name = os.path.join(self.path, "%s.%d-%x" % self.id) def __enter__(self): return self.acquire() diff --git a/borg/remote.py b/borg/remote.py index 6ddc3f1c..b91a4f95 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -185,7 +185,7 @@ class RemoteRepository: else: raise ValueError('log level missing, fix this code') if testing: - return [sys.executable, '-m', 'borg.archiver', 'serve' ] + opts + self.extra_test_args + return [sys.executable, '-m', 'borg.archiver', 'serve'] + opts + self.extra_test_args else: # pragma: no cover return [args.remote_path, 'serve'] + opts diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d6cf8fbf..d9eede40 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -82,6 +82,7 @@ def cmd(request): exe = 'borg.exe' else: raise ValueError("param must be 'python' or 'binary'") + def exec_fn(*args, **kw): return exec_cmd(*args, exe=exe, fork=True, **kw) return exec_fn @@ -121,6 +122,7 @@ if the directory does not exist, the test will be skipped. """ DF_MOUNT = '/tmp/borg-mount' + @pytest.mark.skipif(not os.path.exists(DF_MOUNT), reason="needs a 16MB fs mounted on %s" % DF_MOUNT) def test_disk_full(cmd): def make_files(dir, count, size, rnd=True): @@ -177,7 +179,7 @@ def test_disk_full(cmd): shutil.rmtree(reserve, ignore_errors=True) rc, out = cmd('list', repo) if rc != EXIT_SUCCESS: - print('list', rc, out) + print('list', rc, out) rc, out = cmd('check', '--repair', repo) if rc != EXIT_SUCCESS: print('check', rc, out) @@ -301,7 +303,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): list_output = self.cmd('list', '--short', self.repository_location) self.assert_in('test', list_output) self.assert_in('test.2', list_output) - expected = [ + expected = [ 'input', 'input/bdev', 'input/cdev', @@ -320,7 +322,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): expected.remove('input/cdev') if has_lchflags: # remove the file we did not backup, so input and output become equal - expected.remove('input/flagfile') # this file is UF_NODUMP + expected.remove('input/flagfile') # this file is UF_NODUMP os.remove(os.path.join('input', 'flagfile')) list_output = self.cmd('list', '--short', self.repository_location + '::test') for name in expected: @@ -348,7 +350,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(filter(info_output), filter(info_output2)) def test_atime(self): - have_root = self.create_test_files() + self.create_test_files() atime, mtime = 123456780, 234567890 os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) @@ -414,7 +416,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): filenames = ['normal', 'with some blanks', '(with_parens)', ] for filename in filenames: filename = os.path.join(self.input_path, filename) - with open(filename, 'wb') as fd: + with open(filename, 'wb'): pass self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') @@ -617,11 +619,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.create_regular_file('tagged1/file1', size=1024) self.create_regular_file('tagged2/.NOBACKUP2') self.create_regular_file('tagged2/file2', size=1024) - self.create_regular_file('tagged3/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') + self.create_regular_file('tagged3/CACHEDIR.TAG', contents=b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') self.create_regular_file('tagged3/file3', size=1024) self.create_regular_file('taggedall/.NOBACKUP1') self.create_regular_file('taggedall/.NOBACKUP2') - self.create_regular_file('taggedall/CACHEDIR.TAG', contents = b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') + self.create_regular_file('taggedall/CACHEDIR.TAG', contents=b'Signature: 8a477f597d28d172789f06886806bc55 extra stuff') self.create_regular_file('taggedall/file4', size=1024) self.cmd('create', '--exclude-if-present', '.NOBACKUP1', '--exclude-if-present', '.NOBACKUP2', '--exclude-caches', '--keep-tag-files', self.repository_location + '::test', 'input') @@ -785,7 +787,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): clearly incomplete: only tests for the weird "unchanged" status for now""" now = time.time() self.create_regular_file('file1', size=1024 * 80) - os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago + os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago self.create_regular_file('file2', size=1024 * 80) self.cmd('init', self.repository_location) output = self.cmd('create', '-v', '--list', self.repository_location + '::test', 'input') @@ -822,7 +824,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): output = self.cmd('create', '-v', '--list', '--filter=AM', self.repository_location + '::test3', 'input') self.assert_in('file1', output) - #def test_cmdline_compatibility(self): + # def test_cmdline_compatibility(self): # self.create_regular_file('file1', size=1024 * 80) # self.cmd('init', self.repository_location) # self.cmd('create', self.repository_location + '::test', 'input') diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py index 447a158e..9751bc1a 100644 --- a/borg/testsuite/benchmark.py +++ b/borg/testsuite/benchmark.py @@ -38,12 +38,14 @@ def testdata(request, tmpdir_factory): data_type = request.param if data_type == 'zeros': # do not use a binary zero (\0) to avoid sparse detection - data = lambda: b'0' * size + def data(size): + return b'0' * size if data_type == 'random': - data = lambda: os.urandom(size) + def data(size): + return os.urandom(size) for i in range(count): with open(str(p.join(str(i))), "wb") as f: - f.write(data()) + f.write(data(size)) yield str(p) p.remove(rec=1) @@ -95,4 +97,3 @@ def test_check(benchmark, cmd, archive): def test_help(benchmark, cmd): result, out = benchmark(cmd, 'help') assert result == 0 - diff --git a/borg/testsuite/compress.py b/borg/testsuite/compress.py index ce46c9d3..1a435358 100644 --- a/borg/testsuite/compress.py +++ b/borg/testsuite/compress.py @@ -98,5 +98,3 @@ def test_compressor(): for params in params_list: c = Compressor(**params) assert data == c.decompress(c.compress(data)) - - diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 0d68fe97..a1b5440a 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -165,8 +165,7 @@ class FormatTimedeltaTestCase(BaseTestCase): def check_patterns(files, pattern, expected): """Utility for testing patterns. """ - assert all([f == os.path.normpath(f) for f in files]), \ - "Pattern matchers expect normalized input paths" + assert all([f == os.path.normpath(f) for f in files]), "Pattern matchers expect normalized input paths" matched = [f for f in files if pattern.match(f)] @@ -284,7 +283,7 @@ def test_patterns_shell(pattern, expected): ("^[^/]", []), ("^(?!/srv|/foo|/opt)", ["/home", "/home/user/.profile", "/home/user/.bashrc", "/home/user2/.profile", - "/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails",]), + "/home/user2/public_html/index.html", "/home/foo/.thumbnails", "/home/foo/bar/.thumbnails", ]), ]) def test_patterns_regex(pattern, expected): files = [ diff --git a/borg/testsuite/locking.py b/borg/testsuite/locking.py index dc9d969c..bc62650d 100644 --- a/borg/testsuite/locking.py +++ b/borg/testsuite/locking.py @@ -9,6 +9,7 @@ from ..locking import get_id, TimeoutTimer, ExclusiveLock, UpgradableLock, LockR ID1 = "foo", 1, 1 ID2 = "bar", 2, 2 + def test_id(): hostname, pid, tid = get_id() assert isinstance(hostname, str) diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 3da7b80f..0606280e 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -338,7 +338,7 @@ class RemoteRepositoryTestCase(RepositoryTestCase): remote_path = 'borg' umask = 0o077 - assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve' ] + assert self.repository.borg_cmd(None, testing=True) == [sys.executable, '-m', 'borg.archiver', 'serve'] args = MockArgs() # note: test logger is on info log level, so --info gets added automagically assert self.repository.borg_cmd(args, testing=False) == ['borg', 'serve', '--umask=077', '--info'] diff --git a/borg/upgrader.py b/borg/upgrader.py index 8034661a..e739e071 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -321,6 +321,6 @@ class Borg0xxKeyfileKey(KeyfileKey): filename = os.path.join(keys_dir, name) with open(filename, 'r') as fd: line = fd.readline().strip() - if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID)+1:] == id: + if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID) + 1:] == id: return filename raise KeyfileNotFoundError(repository.path, keys_dir) diff --git a/borg/xattr.py b/borg/xattr.py index c3e85492..27a18df6 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -231,8 +231,8 @@ elif sys.platform.startswith('freebsd'): # pragma: freebsd only mv = memoryview(namebuf.raw) while mv: length = mv[0] - names.append(os.fsdecode(bytes(mv[1:1+length]))) - mv = mv[1+length:] + names.append(os.fsdecode(bytes(mv[1:1 + length]))) + mv = mv[1 + length:] return names def getxattr(path, name, *, follow_symlinks=True): diff --git a/setup.py b/setup.py index 2ba2de47..b64f5bb3 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ on_rtd = os.environ.get('READTHEDOCS') # msgpack pure python data corruption was fixed in 0.4.6. # Also, we might use some rather recent API features. -install_requires=['msgpack-python>=0.4.6', ] +install_requires = ['msgpack-python>=0.4.6', ] from setuptools import setup, Extension @@ -120,12 +120,14 @@ elif not on_rtd: with open('README.rst', 'r') as fd: long_description = fd.read() + class build_usage(Command): description = "generate usage for each command" user_options = [ ('output=', 'O', 'output directory'), ] + def initialize_options(self): pass @@ -172,6 +174,7 @@ class build_api(Command): user_options = [ ('output=', 'O', 'output directory'), ] + def initialize_options(self): pass From 2a2362fc2fce52bd5ede01a081509c871153566b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 22:01:27 +0100 Subject: [PATCH 315/321] add flake8 style checking --- setup.cfg | 10 ++++++++-- tox.ini | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index ecb8cdc1..812f6bee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,5 +2,11 @@ python_files = testsuite/*.py [flake8] -max-line-length = 120 -exclude = build,dist,.git,.idea,.cache,.tox +# please note that the values are adjusted so that they do not cause failures +# with existing code. if you want to change them, you should first fix all +# flake8 failures that appear with your change. +ignore = E122,E123,E125,E126,E127,E128,E226,E402,F401,F811 +# line length long term target: 120 +max-line-length = 255 +exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py + diff --git a/tox.ini b/tox.ini index bc9e6db8..0473cb27 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ # fakeroot -u tox --recreate [tox] -envlist = py{34,35} +envlist = py{34,35},flake8 [testenv] # Change dir to avoid import problem for cython code. The directory does @@ -14,3 +14,8 @@ deps = commands = py.test --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = * + +[testenv:flake8] +changedir = +deps = flake8 +commands = flake8 From 47e630471116b0bd3ebbee54a5af02d994e9874d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 22:03:09 +0100 Subject: [PATCH 316/321] add flake8 tox env to travis config --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 5b46928f..0ec266ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,9 @@ matrix: - python: 3.5 os: linux env: TOXENV=py35 + - python: 3.5 + os: linux + env: TOXENV=flake8 - language: generic os: osx osx_image: xcode6.4 From a65b7ec3397fd838ad75c75345ed19272b9fd9b0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 30 Jan 2016 23:05:40 +0100 Subject: [PATCH 317/321] updates CHANGES --- docs/changes.rst | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index ccbb744d..818fb866 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,8 +5,8 @@ Version 1.0.0 (not released yet) -------------------------------- The major release number change (0.x -> 1.x) indicates bigger incompatible -changes, please read the compatibility notes carefully and adapt / test your -scripts and carefully check your backup logs. +changes, please read the compatibility notes, adapt / test your scripts and +check your backup logs. Compatibility notes: @@ -14,6 +14,8 @@ Compatibility notes: note: we provide binaries that include python 3.5.1 and everything else needed. they are an option in case you are stuck with < 3.4 otherwise. - change encryption to be on by default (using "repokey" mode) +- moved keyfile keys from ~/.borg/keys to ~/.config/borg/keys, + you can either move them manually or run "borg upgrade " - remove support for --encryption=passphrase, use borg migrate-to-repokey to switch to repokey mode, #97 - remove deprecated "--compression ", @@ -61,6 +63,16 @@ Compatibility notes: New features: - borg migrate-to-repokey ("passphrase" -> "repokey" encryption key mode) +- implement --short for borg list REPO, fixes #611 +- implement --list for borg extract (consistency with borg create) +- borg serve: overwrite client's --restrict-to-path with ssh forced command's + option value (but keep everything else from the client commandline), #544 +- use $XDG_CONFIG_HOME/keys for keyfile keys (~/.config/borg/keys), #515 +- "borg upgrade" moves the keyfile keys to the new location + +Bug fixes: + +- normalize trailing slashes for the repository path, #606 Other changes: @@ -78,6 +90,15 @@ Other changes: - use "mock" library from stdlib, #145 - remove borg.support (with non-broken argparse copy), it is ok in 3.4+, #358 - Vagrant: copy CHANGES.rst as symlink, #592 +- cosmetic code cleanups, add flake8 to tox/travis, #4 +- docs / help: + + - make "borg -h" output prettier, #591 + - slightly rephrase prune help + - add missing example for --list option of borg create + - quote exclude line that includes an asterisk to prevent shell expansion + - fix dead link to license + - delete Ubuntu Vivid, it is not supported anymore (EOL) Version 0.30.0 From 57b0ab747596bc92b364bbd896097e9377c7b1af Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 31 Jan 2016 11:50:06 +0100 Subject: [PATCH 318/321] Fixed spelling in deployment docs. --- docs/deployment.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index cffb0831..97ddb5f2 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -69,7 +69,7 @@ forced command and restrictions applied as shown below: The options which are added to the key will perform the following: -1. Force command on the ssh key and dont allow any other command to run +1. Force command on the ssh key and don't allow any other command to run 2. Change working directory 3. Run ``borg serve`` restricted at the client base path 4. Restrict ssh and do not allow stuff which imposes a security risk @@ -111,7 +111,7 @@ Ansible Ansible takes care of all the system-specific commands to add the user, create the folder. Even when the configuration is changed the repository server configuration is -satisfied and reproducable. +satisfied and reproducible. Automate setting up an repository server with the user, group, folders and permissions a Ansible playbook could be used. Keep in mind the playbook @@ -158,7 +158,7 @@ autodetect it runs under SSH by checking the `SSH_ORIGINAL_COMMAND` environment variable. This is left open for future improvements. When extending ssh autodetection in borg no external wrapper script is necessary -and no other interpreter or apllication has to be deployed. +and no other interpreter or application has to be deployed. See also -------- From a7ed46139494c0473bc938044d999361100bb2a4 Mon Sep 17 00:00:00 2001 From: Robin Schneider Date: Sun, 31 Jan 2016 11:51:59 +0100 Subject: [PATCH 319/321] Use `--one-file-system` instead of `--do-not-cross-mountpoints` in docs. Related to #296 in which support for `--do-not-cross-mountpoints` has been deprecated. --- docs/usage.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 22e783f3..fa221b69 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -251,7 +251,7 @@ Examples # Backup the root filesystem into an archive named "root-YYYY-MM-DD" # use zlib compression (good, but slow) - default is no compression NAME="root-`date +%Y-%m-%d`" - $ borg create -C zlib,6 /mnt/backup::$NAME / --do-not-cross-mountpoints + $ borg create -C zlib,6 /mnt/backup::$NAME / --one-file-system # Make a big effort in fine granular deduplication (big chunk management # overhead, needs a lot of RAM and disk space, see formula in internals @@ -381,7 +381,7 @@ Examples Hostname: myhostname Username: root Time: Fri Aug 2 15:18:17 2013 - Command line: /usr/bin/borg create --stats -C zlib,6 /mnt/backup::root-2013-08-02 / --do-not-cross-mountpoints + Command line: /usr/bin/borg create --stats -C zlib,6 /mnt/backup::root-2013-08-02 / --one-file-system Number of files: 147429 Original size: 5344169493 (4.98 GB) Compressed size: 1748189642 (1.63 GB) From 0a0f483daafe4a39be88315333a07df26382c662 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 1 Feb 2016 01:18:37 +0100 Subject: [PATCH 320/321] docs about borg serve's special support for forced/original ssh commands, fixes #544 --- docs/deployment.rst | 7 +++---- docs/usage.rst | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index 97ddb5f2..7349b9bd 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -69,10 +69,9 @@ forced command and restrictions applied as shown below: The options which are added to the key will perform the following: -1. Force command on the ssh key and don't allow any other command to run -2. Change working directory -3. Run ``borg serve`` restricted at the client base path -4. Restrict ssh and do not allow stuff which imposes a security risk +1. Change working directory +2. Run ``borg serve`` restricted to the client base path +3. Restrict ssh and do not allow stuff which imposes a security risk Due to the ``cd`` command we use, the server automatically changes the current working directory. Then client doesn't need to have knowledge of the absolute diff --git a/docs/usage.rst b/docs/usage.rst index fa221b69..519834d7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -426,6 +426,15 @@ Examples Examples ~~~~~~~~ + +borg serve has special support for ssh forced commands (see ``authorized_keys`` +example below): it will detect that you use such a forced command and extract +the value of the ``--restrict-to-path`` option(s). +It will then parse the original command that came from the client, makes sure +that it is also ``borg serve`` and enforce path restriction(s) as given by the +forced command. That way, other options given by the client (like ``--info`` or +``--umask``) are preserved (and are not fixed by the forced command). + :: # Allow an SSH keypair to only run borg, and only have access to /mnt/backup. From 435d30d61b0fdcbe3860ae2b94cc58d61637274b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 1 Feb 2016 03:22:02 +0100 Subject: [PATCH 321/321] docs: updates and fixes --- docs/development.rst | 31 +++++++++++++++---------------- docs/faq.rst | 32 ++++++++++++++++++++++++++++---- docs/installation.rst | 4 +++- docs/internals.rst | 13 +++++-------- docs/quickstart.rst | 7 +++---- docs/support.rst | 4 ++-- 6 files changed, 56 insertions(+), 35 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 3a119218..132dc74c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -15,9 +15,8 @@ Style guide We generally follow `pep8 `_, with 120 columns instead of 79. We do *not* use form-feed (``^L``) characters to -separate sections either. The `flake8 -`_ commandline tool should be used to -check for style errors before sending pull requests. +separate sections either. Compliance is tested automatically when +you run the tests. Output and Logging ------------------ @@ -29,7 +28,7 @@ When directly talking to the user (e.g. Y/N questions), do not use logging, but directly output to stderr (not: stdout, it could be connected to a pipe). To control the amount and kinds of messages output to stderr or emitted at -info level, use flags like --stats. +info level, use flags like ``--stats`` or ``--list``. Building a development environment ---------------------------------- @@ -70,7 +69,7 @@ Some more advanced examples:: Important notes: -- When using -- to give options to py.test, you MUST also give borg.testsuite[.module]. +- When using ``--`` to give options to py.test, you MUST also give ``borg.testsuite[.module]``. Regenerate usage files @@ -120,16 +119,16 @@ The plugin `vagrant-scp` is useful to copy stuff from the VMs to the host. Usage:: - To create and provision the VM: - vagrant up OS - To create an ssh session to the VM: - vagrant ssh OS command - To shut down the VM: - vagrant halt OS - To shut down and destroy the VM: - vagrant destroy OS - To copy files from the VM (in this case, the generated binary): - vagrant scp OS:/vagrant/borg/borg.exe . + # To create and provision the VM: + vagrant up OS + # To create an ssh session to the VM: + vagrant ssh OS command + # To shut down the VM: + vagrant halt OS + # To shut down and destroy the VM: + vagrant destroy OS + # To copy files from the VM (in this case, the generated binary): + vagrant scp OS:/vagrant/borg/borg.exe . Creating standalone binaries @@ -140,7 +139,7 @@ When using the Vagrant VMs, pyinstaller will already be installed. With virtual env activated:: - pip install pyinstaller>=3.0 # or git checkout master + pip install pyinstaller # or git checkout master pyinstaller -F -n borg-PLATFORM borg/__main__.py for file in dist/borg-*; do gpg --armor --detach-sign $file; done diff --git a/docs/faq.rst b/docs/faq.rst index 1831bee6..70322d14 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -26,6 +26,26 @@ repository is only modified from one place. Also keep in mind that |project_name| will keep an exclusive lock on the repository while creating or deleting archives, which may make *simultaneous* backups fail. +Can I copy or synchronize my repo to another location? +------------------------------------------------------ + +Yes, you could just copy all the files. Make sure you do that while no +backup is running. So what you get here is this: + +- client machine ---borg create---> repo1 +- repo1 ---copy---> repo2 + +There is no special borg command to do the copying, just use cp or rsync if +you want to do that. + +But think about whether that is really what you want. If something goes +wrong in repo1, you will have the same issue in repo2 after the copy. + +If you want to have 2 independent backups, it is better to do it like this: + +- client machine ---borg create---> repo1 +- client machine ---borg create---> repo2 + Which file types, attributes, etc. are preserved? ------------------------------------------------- @@ -37,7 +57,7 @@ Which file types, attributes, etc. are preserved? * FIFOs ("named pipes") * Name * Contents - * Time of last modification (nanosecond precision) + * Timestamps in nanosecond precision: mtime, atime, ctime * IDs of owning user and owning group * Names of owning user and owning group (if the IDs can be resolved) * Unix Mode/Permissions (u/g/o permissions, suid, sgid, sticky) @@ -57,6 +77,7 @@ Which file types, attributes, etc. are *not* preserved? backed up as (deduplicated and compressed) runs of zero bytes. Archive extraction has optional support to extract all-zero chunks as holes in a sparse file. + * filesystem specific attributes, like ext4 immutable bit, see :issue:`618`. Why is my backup bigger than with attic? Why doesn't |project_name| do compression by default? ---------------------------------------------------------------------------------------------- @@ -115,9 +136,12 @@ The borg cache eats way too much disk space, what can I do? ----------------------------------------------------------- There is a temporary (but maybe long lived) hack to avoid using lots of disk -space for chunks.archive.d (see issue #235 for details): +space for chunks.archive.d (see :issue:`235` for details): - # this assumes you are working with the same user as the backup +:: + + # this assumes you are working with the same user as the backup. + # you can get the REPOID from the "config" file inside the repository. cd ~/.cache/borg/ rm -rf chunks.archive.d ; touch chunks.archive.d @@ -136,7 +160,7 @@ This has some pros and cons, though: machine (if you share same backup repo between multiple machines) or if your local chunks cache was lost somehow. -The long term plan to improve this is called "borgception", see ticket #474. +The long term plan to improve this is called "borgception", see :issue:`474`. If a backup stops mid-way, does the already-backed-up data stay there? ---------------------------------------------------------------------- diff --git a/docs/installation.rst b/docs/installation.rst index 2d37d1a0..ff3bf450 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -77,7 +77,9 @@ make borg readable and executable for its users and then you can run ``borg``:: sudo chown root:root /usr/local/bin/borg sudo chmod 755 /usr/local/bin/borg -Note that the binary uses /tmp to unpack |project_name| with all dependencies. It will fail if /tmp has not enough free space or is mounted with the ``noexec`` option. You can change the temporary directory by setting the ``TEMP`` environment variable before running |project_name|. +Note that the binary uses /tmp to unpack |project_name| with all dependencies. +It will fail if /tmp has not enough free space or is mounted with the ``noexec`` option. +You can change the temporary directory by setting the ``TEMP`` environment variable before running |project_name|. If a new version is released, you will have to manually download it and replace the old version using the same steps as shown above. diff --git a/docs/internals.rst b/docs/internals.rst index fbbef987..9d1bbd84 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -60,8 +60,8 @@ created by some other process), lock acquisition fails. The cache lock is usually in `~/.cache/borg/REPOID/lock.*`. The repository lock is in `repository/lock.*`. -In case you run into troubles with the locks, you can just delete the `lock.*` -directory and file IF you first make sure that no |project_name| process is +In case you run into troubles with the locks, you can use the ``borg break-lock`` +command after you first have made sure that no |project_name| process is running on any machine that accesses this resource. Be very careful, the cache or repository might get damaged if multiple processes use it at the same time. @@ -181,14 +181,11 @@ Each item represents a file, directory or other fs item and is stored as an * mode (item type + permissions) * source (for links) * rdev (for devices) -* mtime +* mtime, atime, ctime in nanoseconds * xattrs * acl * bsdfiles -``ctime`` (change time) is not stored because there is no API to set -it and it is reset every time an inode's metadata is changed. - All items are serialized using msgpack and the resulting byte stream is fed into the same chunker algorithm as used for regular file data and turned into deduplicated chunks. The reference to these chunks is then added @@ -278,8 +275,8 @@ buckets. As a consequence the hash is just a start position for a linear search, and if the element is not in the table the index is linearly crossed until an empty bucket is found. -When the hash table is filled to 75%, its size is doubled. When it's -emptied to 25%, its size is halved. So operations on it have a variable +When the hash table is filled to 75%, its size is grown. When it's +emptied to 25%, its size is shrinked. So operations on it have a variable complexity between constant and linear with low factor, and memory overhead varies between 33% and 300%. diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 4e0d7b5c..3793b0bb 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -180,7 +180,7 @@ For automated backups the passphrase can be specified using the :ref:`this note about password environments ` for more information. -.. important:: The repository data is totally inaccessible without the key:** +.. warning:: The repository data is totally inaccessible without the key: Make a backup copy of the key file (``keyfile`` mode) or repo config file (``repokey`` mode) and keep it at a safe place, so you still have the key in case it gets corrupted or lost. @@ -204,9 +204,8 @@ or:: Remote operations over SSH can be automated with SSH keys. You can restrict the use of the SSH keypair by prepending a forced command to the SSH public key in -the remote server's authorized_keys file. Only the forced command will be run -when the key authenticates a connection. This example will start |project_name| in server -mode, and limit the |project_name| server to a specific filesystem path:: +the remote server's `authorized_keys` file. This example will start |project_name| +in server mode and limit it to a specific filesystem path:: command="borg serve --restrict-to-path /mnt/backup",no-pty,no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-user-rc ssh-rsa AAAAB3[...] diff --git a/docs/support.rst b/docs/support.rst index 02024b66..1547c666 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -4,8 +4,8 @@ Support ======= -Please first read the docs and existing issue tracker issues and mailing -list posts, a lot of stuff is already documented / explained / discussed / +Please first read the docs, the existing issue tracker issues and mailing +list posts -- a lot of stuff is already documented / explained / discussed / filed there. Issue Tracker