From 83586ae09c7196a33e18898d7df55a09dd692e80 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 3 Apr 2016 17:14:43 +0200 Subject: [PATCH 01/13] Make Repository a context manager, use decorators for wrapping withs (Remote)Repository.close() is not a public API anymore, but a private API. It shall not be used from within other classes than Repository or it's tests. The proper way is to use a context manager now. However, for RPC/Remote compatibility with Borg 1.0 it is kept and unchanged. Repositories are not opened by __init__ now anymore, it is done by binding it to a context manager. (This SHOULD be compatible both ways with remote, since opening the repo is handled by a RepositoryServer method) Decorators @with_repository() and @with_archive simplify context manager handling and remove unnecessary indentation. Backported to 1.0-maint --- borg/archiver.py | 280 ++++++++++++++++++----------------- borg/remote.py | 16 +- borg/repository.py | 23 ++- borg/testsuite/__init__.py | 4 + borg/testsuite/archiver.py | 83 ++++++----- borg/testsuite/repository.py | 57 +++---- borg/testsuite/upgrader.py | 50 +++---- borg/upgrader.py | 47 +++--- 8 files changed, 308 insertions(+), 252 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index c9f72c8db..dd2052be7 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -39,18 +39,56 @@ UMASK_DEFAULT = 0o077 DASHES = '-' * 78 -class ToggleAction(argparse.Action): - """argparse action to handle "toggle" flags easily +def argument(args, str_or_bool): + """If bool is passed, return it. If str is passed, retrieve named attribute from args.""" + if isinstance(str_or_bool, str): + return getattr(args, str_or_bool) + return str_or_bool - toggle flags are in the form of ``--foo``, ``--no-foo``. - the ``--no-foo`` argument still needs to be passed to the - ``add_argument()`` call, but it simplifies the ``--no`` - detection. +def with_repository(fake=False, create=False, lock=True, exclusive=False, manifest=True, cache=False): """ - def __call__(self, parser, ns, values, option): - """set the given flag to true unless ``--no`` is passed""" - setattr(ns, self.dest, not option.startswith('--no-')) + Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …) + + If a parameter (where allowed) is a str the attribute named of args is used instead. + :param fake: (str or bool) use None instead of repository, don't do anything else + :param create: create repository + :param lock: lock repository + :param exclusive: (str or bool) lock repository exclusively (for writing) + :param manifest: load manifest and key, pass them as keyword arguments + :param cache: open cache, pass it as keyword argument (implies manifest) + """ + def decorator(method): + @functools.wraps(method) + def wrapper(self, args, **kwargs): + location = args.location # note: 'location' must be always present in args + if argument(args, fake): + return method(self, args, repository=None, **kwargs) + elif location.proto == 'ssh': + repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args) + else: + repository = Repository(location.path, create=create, exclusive=argument(args, exclusive), + lock_wait=self.lock_wait, lock=lock) + with repository: + if manifest or cache: + kwargs['manifest'], kwargs['key'] = Manifest.load(repository) + if cache: + with Cache(repository, kwargs['key'], kwargs['manifest'], + do_files=getattr(args, 'cache_files', False), lock_wait=self.lock_wait) as cache_: + return method(self, args, repository=repository, cache=cache_, **kwargs) + else: + return method(self, args, repository=repository, **kwargs) + return wrapper + return decorator + + +def with_archive(method): + @functools.wraps(method) + def wrapper(self, args, repository, key, manifest, **kwargs): + archive = Archive(repository, key, manifest, args.location.archive, + numeric_owner=getattr(args, 'numeric_owner', False), cache=kwargs.get('cache')) + return method(self, args, repository=repository, manifest=manifest, key=key, archive=archive, **kwargs) + return wrapper class Archiver: @@ -59,14 +97,6 @@ class Archiver: self.exit_code = EXIT_SUCCESS self.lock_wait = lock_wait - def open_repository(self, args, create=False, exclusive=False, lock=True): - location = args.location # note: 'location' must be always present in args - if location.proto == 'ssh': - repository = RemoteRepository(location, create=create, lock_wait=self.lock_wait, lock=lock, args=args) - else: - repository = Repository(location.path, create=create, exclusive=exclusive, lock_wait=self.lock_wait, lock=lock) - return repository - def print_error(self, msg, *args): msg = args and msg % args or msg self.exit_code = EXIT_ERROR @@ -86,10 +116,10 @@ class Archiver: """ return RepositoryServer(restrict_to_paths=args.restrict_to_paths).serve() - def do_init(self, args): + @with_repository(create=True, exclusive=True, manifest=False) + def do_init(self, args, repository): """Initialize an empty repository""" logger.info('Initializing repository at "%s"' % args.location.canonical_path()) - repository = self.open_repository(args, create=True, exclusive=True) key = key_creator(repository, args) manifest = Manifest(key, repository) manifest.key = key @@ -99,9 +129,9 @@ class Archiver: pass return self.exit_code - def do_check(self, args): + @with_repository(exclusive='repair', manifest=False) + def do_check(self, args, repository): """Check repository consistency""" - repository = self.open_repository(args, exclusive=args.repair) if args.repair: msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + @@ -118,16 +148,15 @@ class Archiver: return EXIT_WARNING return EXIT_SUCCESS - def do_change_passphrase(self, args): + @with_repository() + def do_change_passphrase(self, args, repository, manifest, key): """Change repository key file passphrase""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) key.change_passphrase() return EXIT_SUCCESS - def do_migrate_to_repokey(self, args): + @with_repository(manifest=False) + def do_migrate_to_repokey(self, args, repository): """Migrate passphrase -> repokey""" - repository = self.open_repository(args) manifest_data = repository.get(Manifest.MANIFEST_ID) key_old = PassphraseKey.detect(repository, manifest_data) key_new = RepoKey(repository) @@ -140,7 +169,8 @@ class Archiver: key_new.change_passphrase() # option to change key protection passphrase, save return EXIT_SUCCESS - def do_create(self, args): + @with_repository(fake='dry_run') + def do_create(self, args, repository, manifest=None, key=None): """Create new archive""" matcher = PatternMatcher(fallback=True) if args.excludes: @@ -204,8 +234,6 @@ class Archiver: dry_run = args.dry_run t0 = datetime.utcnow() if not dry_run: - repository = self.open_repository(args, exclusive=True) - manifest, key = Manifest.load(repository) compr_args = dict(buffer=COMPR_BUFFER) compr_args.update(args.compression) key.compressor = Compressor(**compr_args) @@ -298,17 +326,15 @@ class Archiver: status = '-' # dry run, item was not backed up self.print_file_status(status, path) - def do_extract(self, args): + @with_repository() + @with_archive + def do_extract(self, args, repository, manifest, key, archive): """Extract archive contents""" # be restrictive when restoring files, restore permissions later if sys.getfilesystemencoding() == 'ascii': logger.warning('Warning: File system encoding is "ascii", extracting non-ascii filenames will not be supported.') if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, args.location.archive, - numeric_owner=args.numeric_owner) matcher = PatternMatcher() if args.excludes: @@ -359,55 +385,52 @@ class Archiver: self.print_warning("Include pattern '%s' never matched.", pattern) return self.exit_code - def do_rename(self, args): + @with_repository(exclusive=True, cache=True) + @with_archive + def do_rename(self, args, repository, manifest, key, cache, archive): """Rename an existing archive""" - repository = self.open_repository(args, exclusive=True) - manifest, key = Manifest.load(repository) - with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - archive.rename(args.name) - manifest.write() - repository.commit() - cache.commit() + archive.rename(args.name) + manifest.write() + repository.commit() + cache.commit() return self.exit_code - def do_delete(self, args): + @with_repository(exclusive=True, cache=True) + def do_delete(self, args, repository, manifest, key, cache): """Delete an existing repository or archive""" - repository = self.open_repository(args, exclusive=True) - manifest, key = Manifest.load(repository) - with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = Statistics() - archive.delete(stats, progress=args.progress) - manifest.write() - repository.commit(save_space=args.save_space) - cache.commit() - logger.info("Archive deleted.") - if args.stats: - log_multi(DASHES, - stats.summary.format(label='Deleted data:', stats=stats), - str(cache), - DASHES) - else: - if not args.cache_only: - msg = [] - msg.append("You requested to completely DELETE the repository *including* all archives it contains:") - for archive_info in manifest.list_archive_infos(sort_by='ts'): - msg.append(format_archive(archive_info)) - msg.append("Type 'YES' if you understand this and want to continue: ") - msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", truish=('YES', ), - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): - self.exit_code = EXIT_ERROR - return self.exit_code - repository.destroy() - logger.info("Repository deleted.") - cache.destroy() - logger.info("Cache deleted.") + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive, cache=cache) + stats = Statistics() + archive.delete(stats, progress=args.progress) + manifest.write() + repository.commit(save_space=args.save_space) + cache.commit() + logger.info("Archive deleted.") + if args.stats: + log_multi(DASHES, + stats.summary.format(label='Deleted data:', stats=stats), + str(cache), + DASHES) + else: + if not args.cache_only: + msg = [] + msg.append("You requested to completely DELETE the repository *including* all archives it contains:") + for archive_info in manifest.list_archive_infos(sort_by='ts'): + msg.append(format_archive(archive_info)) + msg.append("Type 'YES' if you understand this and want to continue: ") + msg = '\n'.join(msg) + if not yes(msg, false_msg="Aborting.", truish=('YES', ), + env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): + self.exit_code = EXIT_ERROR + return self.exit_code + repository.destroy() + logger.info("Repository deleted.") + cache.destroy() + logger.info("Cache deleted.") return self.exit_code - def do_mount(self, args): + @with_repository() + def do_mount(self, args, repository, manifest, key): """Mount archive or an entire repository as a FUSE fileystem""" try: from .fuse import FuseOperations @@ -419,29 +442,23 @@ class Archiver: self.print_error('%s: Mountpoint must be a writable directory' % args.mountpoint) return self.exit_code - repository = self.open_repository(args) - try: - with cache_if_remote(repository) as cached_repo: - manifest, key = Manifest.load(repository) - if args.location.archive: - archive = Archive(repository, key, manifest, args.location.archive) - else: - archive = None - operations = FuseOperations(key, repository, manifest, archive, cached_repo) - logger.info("Mounting filesystem") - try: - operations.mount(args.mountpoint, args.options, args.foreground) - except RuntimeError: - # Relevant error message already printed to stderr by fuse - self.exit_code = EXIT_ERROR - finally: - repository.close() + with cache_if_remote(repository) as cached_repo: + if args.location.archive: + archive = Archive(repository, key, manifest, args.location.archive) + else: + archive = None + operations = FuseOperations(key, repository, manifest, archive, cached_repo) + logger.info("Mounting filesystem") + try: + operations.mount(args.mountpoint, args.options, args.foreground) + except RuntimeError: + # Relevant error message already printed to stderr by fuse + self.exit_code = EXIT_ERROR return self.exit_code - def do_list(self, args): + @with_repository() + def do_list(self, args, repository, manifest, key): """List archive or repository contents""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) if args.location.archive: archive = Archive(repository, key, manifest, args.location.archive) """use_user_format flag is used to speed up default listing. @@ -515,7 +532,6 @@ class Archiver: item_data['formatkeys'] = list(item_data.keys()) print(format_line(list_format, item_data), end='') - else: for archive_info in manifest.list_archive_infos(sort_by='ts'): if args.prefix and not archive_info.name.startswith(args.prefix): @@ -526,30 +542,27 @@ class Archiver: print(format_archive(archive_info)) return self.exit_code - def do_info(self, args): + @with_repository(cache=True) + @with_archive + def do_info(self, args, repository, manifest, key, archive, cache): """Show archive details such as disk space used""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) - with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: - archive = Archive(repository, key, manifest, args.location.archive, cache=cache) - stats = archive.calc_stats(cache) - print('Name:', archive.name) - print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) - print('Hostname:', archive.metadata[b'hostname']) - print('Username:', archive.metadata[b'username']) - print('Time (start): %s' % format_time(to_localtime(archive.ts))) - print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) - print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) - print('Number of files: %d' % stats.nfiles) - print() - print(str(stats)) - print(str(cache)) + stats = archive.calc_stats(cache) + print('Name:', archive.name) + print('Fingerprint: %s' % hexlify(archive.id).decode('ascii')) + print('Hostname:', archive.metadata[b'hostname']) + print('Username:', archive.metadata[b'username']) + print('Time (start): %s' % format_time(to_localtime(archive.ts))) + print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) + print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) + print('Number of files: %d' % stats.nfiles) + print() + print(str(stats)) + print(str(cache)) return self.exit_code - def do_prune(self, args): + @with_repository() + def do_prune(self, args, repository, manifest, key): """Prune repository archives according to specified rules""" - repository = self.open_repository(args, exclusive=True) - manifest, key = Manifest.load(repository) archives = manifest.list_archive_infos(sort_by='ts', reverse=True) # just a ArchiveInfo list if args.hourly + args.daily + args.weekly + args.monthly + args.yearly == 0 and args.within is None: self.print_error('At least one of the "keep-within", "keep-hourly", "keep-daily", "keep-weekly", ' @@ -614,10 +627,9 @@ class Archiver: print("warning: %s" % e) return self.exit_code - def do_debug_dump_archive_items(self, args): + @with_repository() + def do_debug_dump_archive_items(self, args, repository, manifest, key): """dump (decrypted, decompressed) archive items metadata (not: data)""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) archive = Archive(repository, key, manifest, args.location.archive) for i, item_id in enumerate(archive.metadata[b'items']): data = key.decrypt(item_id, repository.get(item_id)) @@ -628,10 +640,9 @@ class Archiver: print('Done.') return EXIT_SUCCESS - def do_debug_get_obj(self, args): + @with_repository(manifest=False) + def do_debug_get_obj(self, args, repository): """get object contents from the repository and write it into file""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) hex_id = args.id try: id = unhexlify(hex_id) @@ -648,10 +659,9 @@ class Archiver: print("object %s fetched." % hex_id) return EXIT_SUCCESS - def do_debug_put_obj(self, args): + @with_repository(manifest=False) + def do_debug_put_obj(self, args, repository): """put file(s) contents into the repository""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) for path in args.paths: with open(path, "rb") as f: data = f.read() @@ -661,10 +671,9 @@ class Archiver: repository.commit() return EXIT_SUCCESS - def do_debug_delete_obj(self, args): + @with_repository(manifest=False) + def do_debug_delete_obj(self, args, repository): """delete the objects with the given IDs from the repo""" - repository = self.open_repository(args) - manifest, key = Manifest.load(repository) modified = False for hex_id in args.ids: try: @@ -683,14 +692,11 @@ class Archiver: print('Done.') return EXIT_SUCCESS - def do_break_lock(self, args): + @with_repository(lock=False, manifest=False) + def do_break_lock(self, args, repository): """Break the repository lock (e.g. in case it was left by a dead borg.""" - repository = self.open_repository(args, lock=False) - try: - repository.break_lock() - Cache.break_lock(repository) - finally: - repository.close() + repository.break_lock() + Cache.break_lock(repository) return self.exit_code helptext = {} diff --git a/borg/remote.py b/borg/remote.py index b91a4f95e..3fd1cabfe 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -77,6 +77,7 @@ class RepositoryServer: # pragma: no cover if r: data = os.read(stdin_fd, BUFSIZE) if not data: + self.repository.close() return unpacker.feed(data) for unpacked in unpacker: @@ -100,6 +101,7 @@ class RepositoryServer: # pragma: no cover else: os.write(stdout_fd, msgpack.packb((1, msgid, None, res))) if es: + self.repository.close() return def negotiate(self, versions): @@ -117,6 +119,7 @@ class RepositoryServer: # pragma: no cover else: raise PathNotAllowed(path) self.repository = Repository(path, create, lock_wait=lock_wait, lock=lock) + self.repository.__enter__() # clean exit handled by serve() method return self.repository.id @@ -164,11 +167,21 @@ class RemoteRepository: self.id = self.call('open', location.path, create, lock_wait, lock) def __del__(self): - self.close() + if self.p: + self.close() + assert False, "cleanup happened in Repository.__del__" def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path()) + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + self.rollback() + self.close() + def borg_cmd(self, args, testing): """return a borg serve command line""" # give some args/options to "borg serve" process as they were given to us @@ -392,6 +405,7 @@ class RepositoryCache(RepositoryNoCache): super().__init__(repository) tmppath = tempfile.mkdtemp(prefix='borg-tmp') self.caching_repo = Repository(tmppath, create=True, exclusive=True) + self.caching_repo.__enter__() # handled by context manager in base class def close(self): if self.caching_repo is not None: diff --git a/borg/repository.py b/borg/repository.py index 334065bcf..df5610078 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -59,16 +59,31 @@ class Repository: self.lock = None self.index = None self._active_txn = False - if create: - self.create(self.path) - self.open(self.path, exclusive, lock_wait=lock_wait, lock=lock) + self.lock_wait = lock_wait + self.do_lock = lock + self.do_create = create + self.exclusive = exclusive def __del__(self): - self.close() + if self.lock: + self.close() + assert False, "cleanup happened in Repository.__del__" def __repr__(self): return '<%s %s>' % (self.__class__.__name__, self.path) + def __enter__(self): + if self.do_create: + self.do_create = False + self.create(self.path) + self.open(self.path, self.exclusive, lock_wait=self.lock_wait, lock=self.do_lock) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if exc_type is not None: + self.rollback() + self.close() + def create(self, path): """Create a new empty repository at `path` """ diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 1d09be500..5c1a0a6fd 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -8,6 +8,7 @@ import sysconfig import time import unittest from ..xattr import get_all +from ..logger import setup_logging try: import llfuse @@ -30,6 +31,9 @@ else: if sys.platform.startswith('netbsd'): st_mtime_ns_round = -4 # only >1 microsecond resolution here? +# Ensure that the loggers exist for all tests +setup_logging() + class BaseTestCase(unittest.TestCase): """ diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 2ca410a01..6e3862803 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -367,7 +367,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert sto.st_atime_ns == atime * 1e9 def _extract_repository_id(self, path): - return Repository(self.repository_path).id + with Repository(self.repository_path) as repository: + return repository.id def _set_repository_id(self, path, id): config = ConfigParser(interpolation=None) @@ -375,7 +376,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): config.set('repository', 'id', hexlify(id).decode('ascii')) with open(os.path.join(path, 'config'), 'w') as fd: config.write(fd) - return Repository(self.repository_path).id + with Repository(self.repository_path) as repository: + return repository.id def test_sparse_file(self): # no sparse file support on Mac OS X @@ -702,8 +704,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', '--dry-run', self.repository_location + '::test.3') self.cmd('extract', '--dry-run', self.repository_location + '::test.4') # Make sure both archives have been renamed - repository = Repository(self.repository_path) - manifest, key = Manifest.load(repository) + with Repository(self.repository_path) as repository: + manifest, key = Manifest.load(repository) self.assert_equal(len(manifest.archives), 2) self.assert_in('test.3', manifest.archives) self.assert_in('test.4', manifest.archives) @@ -720,8 +722,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', '--dry-run', self.repository_location + '::test.2') self.cmd('delete', '--stats', self.repository_location + '::test.2') # Make sure all data except the manifest has been deleted - repository = Repository(self.repository_path) - self.assert_equal(len(repository), 1) + with Repository(self.repository_path) as repository: + self.assert_equal(len(repository), 1) def test_delete_repo(self): self.create_regular_file('file1', size=1024 * 80) @@ -729,6 +731,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') self.cmd('create', self.repository_location + '::test.2', 'input') + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'no' + self.cmd('delete', self.repository_location, exit_code=2) + self.archiver.exit_code = 0 + assert os.path.exists(self.repository_path) + os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' self.cmd('delete', self.repository_location) # Make sure the repo is gone self.assertFalse(os.path.exists(self.repository_path)) @@ -767,8 +774,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('init', self.repository_location) self.cmd('create', '--dry-run', self.repository_location + '::test', 'input') # Make sure no archive has been created - repository = Repository(self.repository_path) - manifest, key = Manifest.load(repository) + with Repository(self.repository_path) as repository: + manifest, key = Manifest.load(repository) self.assert_equal(len(manifest.archives), 0) def test_progress(self): @@ -968,17 +975,17 @@ class ArchiverTestCase(ArchiverTestCaseBase): used = set() # counter values already used def verify_uniqueness(): - repository = Repository(self.repository_path) - for key, _ in repository.open_index(repository.get_transaction_id()).iteritems(): - data = repository.get(key) - hash = sha256(data).digest() - if hash not in seen: - seen.add(hash) - num_blocks = num_aes_blocks(len(data) - 41) - nonce = bytes_to_long(data[33:41]) - for counter in range(nonce, nonce + num_blocks): - self.assert_not_in(counter, used) - used.add(counter) + with Repository(self.repository_path) as repository: + for key, _ in repository.open_index(repository.get_transaction_id()).iteritems(): + data = repository.get(key) + hash = sha256(data).digest() + if hash not in seen: + seen.add(hash) + num_blocks = num_aes_blocks(len(data) - 41) + nonce = bytes_to_long(data[33:41]) + for counter in range(nonce, nonce + num_blocks): + self.assert_not_in(counter, used) + used.add(counter) self.create_test_files() os.environ['BORG_PASSPHRASE'] = 'passphrase' @@ -1045,8 +1052,9 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): def open_archive(self, name): repository = Repository(self.repository_path) - manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, name) + with repository: + manifest, key = Manifest.load(repository) + archive = Archive(repository, key, manifest, name) return archive, repository def test_check_usage(self): @@ -1064,35 +1072,39 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): def test_missing_file_chunk(self): archive, repository = self.open_archive('archive1') - for item in archive.iter_items(): - if item[b'path'].endswith('testsuite/archiver.py'): - repository.delete(item[b'chunks'][-1][0]) - break - repository.commit() + with repository: + for item in archive.iter_items(): + if item[b'path'].endswith('testsuite/archiver.py'): + repository.delete(item[b'chunks'][-1][0]) + break + repository.commit() self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', '--repair', self.repository_location, exit_code=0) self.cmd('check', self.repository_location, exit_code=0) def test_missing_archive_item_chunk(self): archive, repository = self.open_archive('archive1') - repository.delete(archive.metadata[b'items'][-5]) - repository.commit() + with repository: + repository.delete(archive.metadata[b'items'][-5]) + repository.commit() self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', '--repair', self.repository_location, exit_code=0) self.cmd('check', self.repository_location, exit_code=0) def test_missing_archive_metadata(self): archive, repository = self.open_archive('archive1') - repository.delete(archive.id) - repository.commit() + with repository: + repository.delete(archive.id) + repository.commit() self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', '--repair', self.repository_location, exit_code=0) self.cmd('check', self.repository_location, exit_code=0) def test_missing_manifest(self): archive, repository = self.open_archive('archive1') - repository.delete(Manifest.MANIFEST_ID) - repository.commit() + with repository: + repository.delete(Manifest.MANIFEST_ID) + repository.commit() self.cmd('check', self.repository_location, exit_code=1) output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) self.assert_in('archive1', output) @@ -1101,10 +1113,9 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): def test_extra_chunks(self): self.cmd('check', self.repository_location, exit_code=0) - repository = Repository(self.repository_location) - repository.put(b'01234567890123456789012345678901', b'xxxx') - repository.commit() - repository.close() + with Repository(self.repository_location) as repository: + repository.put(b'01234567890123456789012345678901', b'xxxx') + repository.commit() self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', '--repair', self.repository_location, exit_code=0) diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 0606280ee..f21b40e3c 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -21,6 +21,7 @@ class RepositoryTestCaseBase(BaseTestCase): def setUp(self): self.tmppath = tempfile.mkdtemp() self.repository = self.open(create=True) + self.repository.__enter__() def tearDown(self): self.repository.close() @@ -43,13 +44,12 @@ class RepositoryTestCase(RepositoryTestCaseBase): self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50)) self.repository.commit() self.repository.close() - repository2 = self.open() - self.assert_raises(Repository.ObjectNotFound, lambda: repository2.get(key50)) - for x in range(100): - if x == 50: - continue - self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA') - repository2.close() + with self.open() as repository2: + self.assert_raises(Repository.ObjectNotFound, lambda: repository2.get(key50)) + for x in range(100): + if x == 50: + continue + self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA') def test2(self): """Test multiple sequential transactions @@ -100,13 +100,14 @@ class RepositoryTestCase(RepositoryTestCaseBase): self.repository.close() # replace self.repository = self.open() - self.repository.put(b'00000000000000000000000000000000', b'bar') - self.repository.commit() - self.repository.close() + with self.repository: + self.repository.put(b'00000000000000000000000000000000', b'bar') + self.repository.commit() # delete self.repository = self.open() - self.repository.delete(b'00000000000000000000000000000000') - self.repository.commit() + with self.repository: + self.repository.delete(b'00000000000000000000000000000000') + self.repository.commit() def test_list(self): for x in range(100): @@ -139,8 +140,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): if name.startswith('index.'): os.unlink(os.path.join(self.repository.path, name)) self.reopen() - self.assert_equal(len(self.repository), 3) - self.assert_equal(self.repository.check(), True) + with self.repository: + self.assert_equal(len(self.repository), 3) + self.assert_equal(self.repository.check(), True) def test_crash_before_compact_segments(self): self.add_keys() @@ -150,8 +152,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): except TypeError: pass self.reopen() - self.assert_equal(len(self.repository), 3) - self.assert_equal(self.repository.check(), True) + with self.repository: + self.assert_equal(len(self.repository), 3) + self.assert_equal(self.repository.check(), True) def test_replay_of_readonly_repository(self): self.add_keys() @@ -160,8 +163,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): os.unlink(os.path.join(self.repository.path, name)) with patch.object(UpgradableLock, 'upgrade', side_effect=LockFailed) as upgrade: self.reopen() - self.assert_raises(LockFailed, lambda: len(self.repository)) - upgrade.assert_called_once_with() + with self.repository: + self.assert_raises(LockFailed, lambda: len(self.repository)) + upgrade.assert_called_once_with() def test_crash_before_write_index(self): self.add_keys() @@ -171,8 +175,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): except TypeError: pass self.reopen() - self.assert_equal(len(self.repository), 3) - self.assert_equal(self.repository.check(), True) + with self.repository: + self.assert_equal(len(self.repository), 3) + self.assert_equal(self.repository.check(), True) def test_crash_before_deleting_compacted_segments(self): self.add_keys() @@ -182,9 +187,10 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): except TypeError: pass self.reopen() - self.assert_equal(len(self.repository), 3) - self.assert_equal(self.repository.check(), True) - self.assert_equal(len(self.repository), 3) + with self.repository: + self.assert_equal(len(self.repository), 3) + self.assert_equal(self.repository.check(), True) + self.assert_equal(len(self.repository), 3) class RepositoryCheckTestCase(RepositoryTestCaseBase): @@ -313,8 +319,9 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.repository.commit() compact.assert_called_once_with(save_space=False) self.reopen() - self.check(repair=True) - self.assert_equal(self.repository.get(bytes(32)), b'data2') + with self.repository: + self.check(repair=True) + self.assert_equal(self.repository.get(bytes(32)), b'data2') class RemoteRepositoryTestCase(RepositoryTestCase): diff --git a/borg/testsuite/upgrader.py b/borg/testsuite/upgrader.py index 9a1f823f9..26c34a3c6 100644 --- a/borg/testsuite/upgrader.py +++ b/borg/testsuite/upgrader.py @@ -23,11 +23,9 @@ def repo_valid(path): :param path: the path to the repository :returns: if borg can check the repository """ - repository = Repository(str(path), create=False) - # can't check raises() because check() handles the error - state = repository.check() - repository.close() - return state + with Repository(str(path), create=False) as repository: + # can't check raises() because check() handles the error + return repository.check() def key_valid(path): @@ -79,11 +77,11 @@ def test_convert_segments(tmpdir, attic_repo, inplace): """ # check should fail because of magic number assert not repo_valid(tmpdir) - repo = AtticRepositoryUpgrader(str(tmpdir), create=False) - segments = [filename for i, filename in repo.io.segment_iterator()] - repo.close() - repo.convert_segments(segments, dryrun=False, inplace=inplace) - repo.convert_cache(dryrun=False) + repository = AtticRepositoryUpgrader(str(tmpdir), create=False) + with repository: + segments = [filename for i, filename in repository.io.segment_iterator()] + repository.convert_segments(segments, dryrun=False, inplace=inplace) + repository.convert_cache(dryrun=False) assert repo_valid(tmpdir) @@ -138,9 +136,9 @@ def test_keys(tmpdir, attic_repo, attic_key_file): define above) :param attic_key_file: an attic.key.KeyfileKey (fixture created above) """ - repository = AtticRepositoryUpgrader(str(tmpdir), create=False) - keyfile = AtticKeyfileKey.find_key_file(repository) - AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False) + with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository: + keyfile = AtticKeyfileKey.find_key_file(repository) + AtticRepositoryUpgrader.convert_keyfiles(keyfile, dryrun=False) assert key_valid(attic_key_file.path) @@ -167,19 +165,19 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): return stat_segment(path).st_ino orig_inode = first_inode(attic_repo.path) - repo = AtticRepositoryUpgrader(str(tmpdir), create=False) - # replicate command dispatch, partly - os.umask(UMASK_DEFAULT) - backup = repo.upgrade(dryrun=False, inplace=inplace) - if inplace: - assert backup is None - assert first_inode(repo.path) == orig_inode - else: - assert backup - assert first_inode(repo.path) != first_inode(backup) - # i have seen cases where the copied tree has world-readable - # permissions, which is wrong - assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 + with AtticRepositoryUpgrader(str(tmpdir), create=False) as repository: + # replicate command dispatch, partly + os.umask(UMASK_DEFAULT) + backup = repository.upgrade(dryrun=False, inplace=inplace) + if inplace: + assert backup is None + assert first_inode(repository.path) == orig_inode + else: + assert backup + assert first_inode(repository.path) != first_inode(backup) + # i have seen cases where the copied tree has world-readable + # permissions, which is wrong + assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 assert key_valid(attic_key_file.path) assert repo_valid(tmpdir) diff --git a/borg/upgrader.py b/borg/upgrader.py index e739e0714..75d9fbb46 100644 --- a/borg/upgrader.py +++ b/borg/upgrader.py @@ -30,23 +30,23 @@ class AtticRepositoryUpgrader(Repository): we nevertheless do the order in reverse, as we prefer to do the fast stuff first, to improve interactivity. """ - backup = None - if not inplace: - backup = '{}.upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now()) - logger.info('making a hardlink copy in %s', backup) - if not dryrun: - shutil.copytree(self.path, backup, copy_function=os.link) - logger.info("opening attic repository with borg and converting") - # now lock the repo, after we have made the copy - self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() - segments = [filename for i, filename in self.io.segment_iterator()] - try: - keyfile = self.find_attic_keyfile() - except KeyfileNotFoundError: - logger.warning("no key file found for repository") - else: - self.convert_keyfiles(keyfile, dryrun) - self.close() + with self: + backup = None + if not inplace: + backup = '{}.upgrade-{:%Y-%m-%d-%H:%M:%S}'.format(self.path, datetime.datetime.now()) + logger.info('making a hardlink copy in %s', backup) + if not dryrun: + shutil.copytree(self.path, backup, copy_function=os.link) + logger.info("opening attic repository with borg and converting") + # now lock the repo, after we have made the copy + self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True, timeout=1.0).acquire() + segments = [filename for i, filename in self.io.segment_iterator()] + try: + keyfile = self.find_attic_keyfile() + except KeyfileNotFoundError: + logger.warning("no key file found for repository") + else: + self.convert_keyfiles(keyfile, dryrun) # partial open: just hold on to the lock self.lock = UpgradableLock(os.path.join(self.path, 'lock'), exclusive=True).acquire() @@ -282,12 +282,13 @@ class BorgRepositoryUpgrader(Repository): """convert an old borg repository to a current borg repository """ logger.info("converting borg 0.xx to borg current") - try: - keyfile = self.find_borg0xx_keyfile() - except KeyfileNotFoundError: - logger.warning("no key file found for repository") - else: - self.move_keyfiles(keyfile, dryrun) + with self: + try: + keyfile = self.find_borg0xx_keyfile() + except KeyfileNotFoundError: + logger.warning("no key file found for repository") + else: + self.move_keyfiles(keyfile, dryrun) def find_borg0xx_keyfile(self): return Borg0xxKeyfileKey.find_key_file(self) From 86361fd75f8df638d8ebb3c82f5a434d8f509a2d Mon Sep 17 00:00:00 2001 From: TW Date: Sun, 3 Apr 2016 17:49:37 +0200 Subject: [PATCH 02/13] update CHANGES the password roundtrip was already in 1.0.0, removed it from 1.0.1 changes. rephrase / prettify some stuff. --- docs/changes.rst | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index e0e0d3386..fa732f3f7 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -6,35 +6,32 @@ Version 1.0.1 (not released yet) New features: -Usually there are no new features in a bugfix release, but these 2 were added -to get them out quickly - as both positively affect your safety/security: +Usually there are no new features in a bugfix release, but this one was added +to get it out quickly as it can improve your repository safety/security: -- append-only mode for repositories, #809, #36 - (please read the docs about how this can improve your security) -- implement password roundtrip, #695 (make sure the user can know/verify the - encryption key password/passphrase, to avoid double-typos, wrong keyboard - layout or locale/encoding issues) +- append-only mode for repositories, #809, #36 (see docs) Bug fixes: - fix silently skipping EIO, #748 -- do not sleep for >60s while waiting for lock, fixes #773 +- add context manager for Repository (avoid orphan repository locks), #285 +- do not sleep for >60s while waiting for lock, #773 - unpack file stats before passing to FUSE - fix build on illumos - don't try to backup doors or event ports (Solaris and derivates) - fix capitalization, add ellipses, change log level to debug for 2 messages, fixes #798 -- remove useless/misleading libc version display, fixes #738 +- remove useless/misleading libc version display, #738 Other changes: - update llfuse requirement, llfuse 1.0 works -- update OS / dist packages on build machines, fixes #717 +- update OS / dist packages on build machines, #717 - docs: - fix cygwin requirements (gcc-g++) - - document how to debug / file filesystem issues, fixes #664 + - document how to debug / file filesystem issues, #664 - fix reproducible build of api docs - - RTD theme: CSS !important overwrite. Fix borgbackup/borg#727 + - RTD theme: CSS !important overwrite, #727 - Document logo font. Recreate logo png. Remove GIMP logo file. From d7299c92300ec25cf399e642bcd1090118f0e37e Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 3 Apr 2016 17:58:15 +0200 Subject: [PATCH 03/13] ArchiveChecker: move "orphaned objects check skipped" to INFO log level Fixes #826 --- borg/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index c5d9fb842..94948e89f 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -879,7 +879,7 @@ class ArchiveChecker: for id_ in unused: self.repository.delete(id_) else: - logger.warning('Orphaned objects check skipped (needs all archives checked).') + logger.info('Orphaned objects check skipped (needs all archives checked).') def finish(self, save_space=False): if self.repair: From a5193333766ded9cbc7c94a1e151a6a014a74aa9 Mon Sep 17 00:00:00 2001 From: TW Date: Sun, 3 Apr 2016 20:17:09 +0200 Subject: [PATCH 04/13] add --warning, --error, --critical for completeness, fixes #826 it's not recommended to suppress warnings or errors, but the user may decide this on his own. note: --warning is not given to borg serve so a <= 1.0.0 borg will still work as server. it is not needed as it is the default. --- borg/archiver.py | 13 +++++++++++-- borg/remote.py | 4 ++++ docs/usage.rst | 28 ++++++++++++++++++++-------- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index dd2052be7..a790ea34a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -818,12 +818,21 @@ class Archiver: def build_parser(self, args=None, prog=None): common_parser = argparse.ArgumentParser(add_help=False, prog=prog) + common_parser.add_argument('--critical', dest='log_level', + action='store_const', const='critical', default='warning', + help='work on log level CRITICAL') + common_parser.add_argument('--error', dest='log_level', + action='store_const', const='error', default='warning', + help='work on log level ERROR') + common_parser.add_argument('--warning', dest='log_level', + action='store_const', const='warning', default='warning', + help='work on log level WARNING (default)') common_parser.add_argument('-v', '--verbose', '--info', dest='log_level', action='store_const', const='info', default='warning', - help='enable informative (verbose) output, work on log level INFO') + help='work on log level INFO') common_parser.add_argument('--debug', dest='log_level', action='store_const', const='debug', default='warning', - help='enable debug output, work on log level DEBUG') + help='work on log level DEBUG') common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, help='wait for the lock, but max. N seconds (default: %(default)d).') common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, diff --git a/borg/remote.py b/borg/remote.py index 3fd1cabfe..c81ae7adc 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -195,6 +195,10 @@ class RemoteRepository: opts.append('--info') elif root_logger.isEnabledFor(logging.WARNING): pass # warning is default + elif root_logger.isEnabledFor(logging.ERROR): + opts.append('--error') + elif root_logger.isEnabledFor(logging.CRITICAL): + opts.append('--critical') else: raise ValueError('log level missing, fix this code') if testing: diff --git a/docs/usage.rst b/docs/usage.rst index fc19c6ac4..6a3843f86 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -16,19 +16,31 @@ Type of log output The log level of the builtin logging configuration defaults to WARNING. This is because we want |project_name| to be mostly silent and only output -warnings (plus errors and critical messages). - -Use ``--verbose`` or ``--info`` to set INFO (you will get informative output then -additionally to warnings, errors, critical messages). -Use ``--debug`` to set DEBUG to get output made for debugging. - -All log messages created with at least the set level will be output. +warnings, errors and critical messages. Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL +Use ``--debug`` to set DEBUG log level - +to get debug, info, warning, error and critical level output. + +Use ``--info`` (or ``-v`` or ``--verbose``) to set INFO log level - +to get info, warning, error and critical level output. + +Use ``--warning`` (default) to set WARNING log level - +to get warning, error and critical level output. + +Use ``--error`` to set ERROR log level - +to get error and critical level output. + +Use ``--critical`` to set CRITICAL log level - +to get critical level output. + While you can set misc. log levels, do not expect that every command will give different output on different log levels - it's just a possibility. +.. warning:: Options --critical and --error are provided for completeness, + their usage is not recommended as you might miss important information. + .. warning:: While some options (like ``--stats`` or ``--list``) will emit more informational messages, you have to use INFO (or lower) log level to make them show up in log output. Use ``-v`` or a logging configuration. @@ -765,4 +777,4 @@ for e.g. regular pruning. Further protections can be implemented, but are outside of Borgs scope. For example, file system snapshots or wrapping ``borg serve`` to set special permissions or ACLs on -new data files. \ No newline at end of file +new data files. From 8f716d8ca1a8632d9dc0458b4a00ffbab77dc98f Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 3 Apr 2016 21:37:15 +0200 Subject: [PATCH 05/13] If BORG_PASSPHRASE is present but wrong, don't prompt for password, fail instead. Leaves PassphraseKey alone, since I cannot test it. Fixes #791 --- borg/key.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/borg/key.py b/borg/key.py index 113214ab6..d52c1fd2d 100644 --- a/borg/key.py +++ b/borg/key.py @@ -18,6 +18,10 @@ import msgpack PREFIX = b'\0' * 8 +class PassphraseWrong(Error): + """passphrase supplied in BORG_PASSPHRASE is incorrect""" + + class PasswordRetriesExceeded(Error): """exceeded the maximum password retries""" @@ -284,13 +288,19 @@ class KeyfileKeyBase(AESKeyBase): key = cls(repository) target = key.find_key() prompt = 'Enter passphrase for key %s: ' % target - passphrase = Passphrase.env_passphrase(default='') - for retry in range(1, 4): - if key.load(target, passphrase): - break - passphrase = Passphrase.getpass(prompt) + passphrase = Passphrase.env_passphrase() + if passphrase is None: + passphrase = Passphrase() + if not key.load(target, passphrase): + for retry in range(0, 3): + passphrase = Passphrase.getpass(prompt) + if key.load(target, passphrase): + break + else: + raise PasswordRetriesExceeded else: - raise PasswordRetriesExceeded + if not key.load(target, passphrase): + raise PassphraseWrong num_blocks = num_aes_blocks(len(manifest_data) - 41) key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks)) return key From 425a4d819d39f2e5f601e2822e7a427eca2d2058 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 4 Apr 2016 17:09:52 +0200 Subject: [PATCH 06/13] Remote: don't print tracebacks for Error exceptions handled downstream (What we'd really want is to have proper exception transporting over the RPC) Fixes #792 --- borg/remote.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index b91a4f95e..af74c69d8 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -93,8 +93,12 @@ class RepositoryServer: # pragma: no cover f = getattr(self.repository, method) res = f(*args) except BaseException as e: - logging.exception('Borg %s: exception in RPC call:', __version__) - logging.error(sysinfo()) + # These exceptions are reconstructed on the client end in RemoteRepository.call_many(), + # and will be handled just like locally raised exceptions. Suppress the remote traceback + # for these, except ErrorWithTraceback, which should always display a traceback. + if not isinstance(e, (Repository.DoesNotExist, Repository.AlreadyExists, PathNotAllowed)): + logging.exception('Borg %s: exception in RPC call:', __version__) + logging.error(sysinfo()) exc = "Remote Exception (see remote log for the traceback)" os.write(stdout_fd, msgpack.packb((1, msgid, e.__class__.__name__, exc))) else: From 890e44ef60a9a90ee6aab60e3ece2b6b3159e0bb Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 3 Apr 2016 21:44:29 +0200 Subject: [PATCH 07/13] RemoteRepository: clean up pipe if remote open() fails --- borg/remote.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/borg/remote.py b/borg/remote.py index af74c69d8..30a343922 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -81,6 +81,7 @@ class RepositoryServer: # pragma: no cover unpacker.feed(data) for unpacked in unpacker: if not (isinstance(unpacked, tuple) and len(unpacked) == 4): + self.repository.close() raise Exception("Unexpected RPC data format.") type, msgid, method, args = unpacked method = method.decode('ascii') @@ -165,7 +166,11 @@ class RemoteRepository: raise ConnectionClosedWithHint('Is borg working on the server?') from None if version != RPC_PROTOCOL_VERSION: raise Exception('Server insisted on using unsupported protocol version %d' % version) - self.id = self.call('open', location.path, create, lock_wait, lock) + try: + self.id = self.call('open', self.location.path, create, lock_wait, lock) + except Exception: + self.close() + raise def __del__(self): self.close() From b59f92a5336862888804c792fb244b9ec3aeacda Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 6 Apr 2016 10:04:35 +0200 Subject: [PATCH 08/13] Test suite: Reset exit code of persistent archiver #844 --- borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6e3862803..1723bd7af 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -59,6 +59,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): sys.stdout = sys.stderr = output = StringIO() if archiver is None: archiver = Archiver() + archiver.exit_code = EXIT_SUCCESS args = archiver.parse_args(list(args)) ret = archiver.run(args) return ret, output.getvalue() @@ -733,7 +734,6 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('create', self.repository_location + '::test.2', 'input') os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'no' self.cmd('delete', self.repository_location, exit_code=2) - self.archiver.exit_code = 0 assert os.path.exists(self.repository_path) os.environ['BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'] = 'YES' self.cmd('delete', self.repository_location) From 29bc3965593dbcb62fa1725776da4a651652f9b0 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 6 Apr 2016 23:17:06 +0200 Subject: [PATCH 09/13] borg create: add --ignore-inode option This is mainly meant for use with network file systems like sshfs and possibly CIFS, which don't convey proper inode numbers. --- borg/archive.py | 4 ++-- borg/archiver.py | 12 +++++++++++- borg/cache.py | 5 +++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 94948e89f..83308e66d 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -517,7 +517,7 @@ Number of files: {0.stats.nfiles}'''.format( self.add_item(item) return 'i' # stdin - def process_file(self, path, st, cache): + def process_file(self, path, st, cache, ignore_inode=False): status = None safe_path = make_path_safe(path) # Is it a hard link? @@ -533,7 +533,7 @@ Number of files: {0.stats.nfiles}'''.format( self.hard_links[st.st_ino, st.st_dev] = safe_path path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape')) first_run = not cache.files - ids = cache.file_known_and_unchanged(path_hash, st) + ids = cache.file_known_and_unchanged(path_hash, st, ignore_inode) if first_run: logger.debug('Processing files ...') chunks = None diff --git a/borg/archiver.py b/borg/archiver.py index a790ea34a..c49747e7c 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -231,6 +231,7 @@ class Archiver: self.output_filter = args.output_filter self.output_list = args.output_list + self.ignore_inode = args.ignore_inode dry_run = args.dry_run t0 = datetime.utcnow() if not dry_run: @@ -270,7 +271,7 @@ class Archiver: if stat.S_ISREG(st.st_mode) or read_special and not stat.S_ISDIR(st.st_mode): if not dry_run: try: - status = archive.process_file(path, st, cache) + status = archive.process_file(path, st, cache, self.ignore_inode) except OSError as e: status = 'E' self.print_warning('%s: %s', path, e) @@ -984,6 +985,12 @@ class Archiver: traversing all paths specified. The archive will consume almost no disk space for files or parts of files that have already been stored in other archives. + + To speed up pulling backups over sshfs and similar network file systems which do + not provide correct inode information the --ignore-inode flag can be used. This + potentially decreases reliability of change detection, while avoiding always reading + all files on these file systems. + See the output of the "borg help patterns" command for more help on exclude patterns. """) @@ -1039,6 +1046,9 @@ class Archiver: type=ChunkerParams, default=CHUNKER_PARAMS, metavar='CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE', help='specify the chunker parameters. default: %d,%d,%d,%d' % CHUNKER_PARAMS) + subparser.add_argument('--ignore-inode', dest='ignore_inode', + action='store_true', default=False, + help='ignore inode data in the file metadata cache used to detect unchanged files.') subparser.add_argument('-C', '--compression', dest='compression', type=CompressionSpec, default=dict(name='none'), metavar='COMPRESSION', help='select compression algorithm (and level): ' diff --git a/borg/cache.py b/borg/cache.py index 79f7bd2d5..3c1e11f22 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -394,7 +394,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" self.chunks[id] = (count - 1, size, csize) stats.update(-size, -csize, False) - def file_known_and_unchanged(self, path_hash, st): + def file_known_and_unchanged(self, path_hash, st, ignore_inode=False): if not (self.do_files and stat.S_ISREG(st.st_mode)): return None if self.files is None: @@ -403,7 +403,8 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not entry: return None entry = msgpack.unpackb(entry) - if entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and entry[1] == st.st_ino: + if (entry[2] == st.st_size and bigint_to_int(entry[3]) == st.st_mtime_ns and + (ignore_inode or entry[1] == st.st_ino)): # reset entry age entry[0] = 0 self.files[path_hash] = msgpack.packb(entry) From e9b3b3fa1a8cd0f293c227890e489e7c5fc4a8c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 8 Apr 2016 19:52:47 +0200 Subject: [PATCH 10/13] updated CHANGES --- docs/changes.rst | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index fa732f3f7..a4cc750e3 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -6,10 +6,20 @@ Version 1.0.1 (not released yet) New features: -Usually there are no new features in a bugfix release, but this one was added -to get it out quickly as it can improve your repository safety/security: +Usually there are no new features in a bugfix release, but these were added +due to their high impact on security/safety/speed or because they are fixes +also: - append-only mode for repositories, #809, #36 (see docs) +- borg create: add --ignore-inode option to make borg detect unmodified files + even if your filesystem does not have stable inode numbers (like sshfs and + possibly CIFS). +- add options --warning, --error, --critical for missing log levels, #826. + it's not recommended to suppress warnings or errors, but the user may decide + this on his own. + note: --warning is not given to borg serve so a <= 1.0.0 borg will still + work as server (it is not needed as it is the default). + do not use --error or --critical when using a <= 1.0.0 borg server. Bug fixes: @@ -19,8 +29,15 @@ Bug fixes: - unpack file stats before passing to FUSE - fix build on illumos - don't try to backup doors or event ports (Solaris and derivates) -- fix capitalization, add ellipses, change log level to debug for 2 messages, fixes #798 - remove useless/misleading libc version display, #738 +- test suite: reset exit code of persistent archiver, #844 +- RemoteRepository: clean up pipe if remote open() fails +- Remote: don't print tracebacks for Error exceptions handled downstream, #792 +- if BORG_PASSPHRASE is present but wrong, don't prompt for password, but fail + instead, #791 +- ArchiveChecker: move "orphaned objects check skipped" to INFO log level, #826 +- fix capitalization, add ellipses, change log level to debug for 2 messages, + #798 Other changes: From 7861de3930a8a1fcb32ec72dcdad2a5ab92fa2fd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 8 Apr 2016 22:55:07 +0200 Subject: [PATCH 11/13] reorder log level options, so --info shows up in help --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index c49747e7c..7f7bcbc9a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -828,7 +828,7 @@ class Archiver: common_parser.add_argument('--warning', dest='log_level', action='store_const', const='warning', default='warning', help='work on log level WARNING (default)') - common_parser.add_argument('-v', '--verbose', '--info', dest='log_level', + common_parser.add_argument('--info', '-v', '--verbose', dest='log_level', action='store_const', const='info', default='warning', help='work on log level INFO') common_parser.add_argument('--debug', dest='log_level', From d5d037b97d7a24e0e3d2375d5d489dba5e99b5dd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 8 Apr 2016 23:38:46 +0200 Subject: [PATCH 12/13] ran build_api build_usage --- docs/api.rst | 80 ++++++++++----------- docs/usage/break-lock.rst.inc | 13 ++-- docs/usage/change-passphrase.rst.inc | 13 ++-- docs/usage/check.rst.inc | 18 ++--- docs/usage/create.rst.inc | 27 ++++--- docs/usage/debug-delete-obj.rst.inc | 13 ++-- docs/usage/debug-dump-archive-items.rst.inc | 13 ++-- docs/usage/debug-get-obj.rst.inc | 13 ++-- docs/usage/debug-put-obj.rst.inc | 13 ++-- docs/usage/delete.rst.inc | 16 +++-- docs/usage/extract.rst.inc | 20 +++--- docs/usage/info.rst.inc | 15 ++-- docs/usage/init.rst.inc | 16 +++-- docs/usage/list.rst.inc | 17 +++-- docs/usage/migrate-to-repokey.rst.inc | 13 ++-- docs/usage/mount.rst.inc | 16 +++-- docs/usage/prune.rst.inc | 20 +++--- docs/usage/rename.rst.inc | 15 ++-- docs/usage/serve.rst.inc | 16 +++-- docs/usage/upgrade.rst.inc | 16 +++-- 20 files changed, 220 insertions(+), 163 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index 628d21d13..7f56c4f77 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -2,43 +2,15 @@ API Documentation ================= -.. automodule:: borg.archiver - :members: - :undoc-members: - -.. automodule:: borg.upgrader - :members: - :undoc-members: - -.. automodule:: borg.archive - :members: - :undoc-members: - -.. automodule:: borg.fuse - :members: - :undoc-members: - .. automodule:: borg.platform :members: :undoc-members: -.. automodule:: borg.locking - :members: - :undoc-members: - .. automodule:: borg.shellpattern :members: :undoc-members: -.. automodule:: borg.repository - :members: - :undoc-members: - -.. automodule:: borg.lrucache - :members: - :undoc-members: - -.. automodule:: borg.remote +.. automodule:: borg.locking :members: :undoc-members: @@ -46,7 +18,19 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.xattr +.. automodule:: borg.logger + :members: + :undoc-members: + +.. automodule:: borg.remote + :members: + :undoc-members: + +.. automodule:: borg.fuse + :members: + :undoc-members: + +.. automodule:: borg.archive :members: :undoc-members: @@ -54,6 +38,22 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.lrucache + :members: + :undoc-members: + +.. automodule:: borg.xattr + :members: + :undoc-members: + +.. automodule:: borg.archiver + :members: + :undoc-members: + +.. automodule:: borg.repository + :members: + :undoc-members: + .. automodule:: borg.cache :members: :undoc-members: @@ -62,7 +62,7 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.logger +.. automodule:: borg.upgrader :members: :undoc-members: @@ -70,19 +70,11 @@ API Documentation :members: :undoc-members: -.. automodule:: borg.platform_linux - :members: - :undoc-members: - -.. automodule:: borg.hashindex - :members: - :undoc-members: - .. automodule:: borg.compress :members: :undoc-members: -.. automodule:: borg.chunker +.. automodule:: borg.platform_linux :members: :undoc-members: @@ -90,6 +82,14 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.chunker + :members: + :undoc-members: + .. automodule:: borg.platform_freebsd :members: :undoc-members: + +.. automodule:: borg.hashindex + :members: + :undoc-members: diff --git a/docs/usage/break-lock.rst.inc b/docs/usage/break-lock.rst.inc index d59b1dc05..4edef5c23 100644 --- a/docs/usage/break-lock.rst.inc +++ b/docs/usage/break-lock.rst.inc @@ -4,7 +4,8 @@ borg break-lock --------------- :: - usage: borg break-lock [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg break-lock [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] REPOSITORY @@ -15,10 +16,12 @@ borg break-lock optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index eb52399c3..08ba4ee9f 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -4,7 +4,8 @@ borg change-passphrase ---------------------- :: - usage: borg change-passphrase [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg change-passphrase [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] @@ -16,10 +17,12 @@ borg change-passphrase optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index b22d75383..9ad6a4b35 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -4,10 +4,10 @@ borg check ---------- :: - usage: borg check [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [--repository-only] [--archives-only] [--repair] - [--save-space] [--last N] [-P PREFIX] + usage: borg check [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--repository-only] [--archives-only] + [--repair] [--save-space] [--last N] [-P PREFIX] [REPOSITORY_OR_ARCHIVE] Check repository consistency @@ -18,10 +18,12 @@ borg check optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 6ec3e7de7..ec9e7dc66 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -4,15 +4,16 @@ borg create ----------- :: - usage: borg create [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-s] - [-p] [--list] [--filter STATUSCHARS] [-e PATTERN] + usage: borg create [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-s] [-p] [--list] + [--filter STATUSCHARS] [-e PATTERN] [--exclude-from EXCLUDEFILE] [--exclude-caches] [--exclude-if-present FILENAME] [--keep-tag-files] [-c SECONDS] [-x] [--numeric-owner] [--timestamp yyyy-mm-ddThh:mm:ss] [--chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE] - [-C COMPRESSION] [--read-special] [-n] + [--ignore-inode] [-C COMPRESSION] [--read-special] [-n] ARCHIVE PATH [PATH ...] Create new archive @@ -24,10 +25,12 @@ borg create optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to @@ -60,6 +63,8 @@ borg create alternatively, give a reference file/directory. --chunker-params CHUNK_MIN_EXP,CHUNK_MAX_EXP,HASH_MASK_BITS,HASH_WINDOW_SIZE specify the chunker parameters. default: 19,23,21,4095 + --ignore-inode ignore inode data in the file metadata cache used to + detect unchanged files. -C COMPRESSION, --compression COMPRESSION select compression algorithm (and level): none == no compression (default), lz4 == lz4, zlib == zlib @@ -77,4 +82,10 @@ This command creates a backup archive containing all files found while recursive traversing all paths specified. The archive will consume almost no disk space for files or parts of files that have already been stored in other archives. + +To speed up pulling backups over sshfs and similar network file systems which do +not provide correct inode information the --ignore-inode flag can be used. This +potentially decreases reliability of change detection, while avoiding always reading +all files on these file systems. + See the output of the "borg help patterns" command for more help on exclude patterns. diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc index b02d7b721..bfc31043a 100644 --- a/docs/usage/debug-delete-obj.rst.inc +++ b/docs/usage/debug-delete-obj.rst.inc @@ -4,7 +4,8 @@ borg debug-delete-obj --------------------- :: - usage: borg debug-delete-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg debug-delete-obj [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] IDs [IDs ...] @@ -17,10 +18,12 @@ borg debug-delete-obj optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc index 9265f2c09..fd2cd62c9 100644 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ b/docs/usage/debug-dump-archive-items.rst.inc @@ -4,7 +4,8 @@ borg debug-dump-archive-items ----------------------------- :: - usage: borg debug-dump-archive-items [-h] [-v] [--debug] [--lock-wait N] + usage: borg debug-dump-archive-items [-h] [--critical] [--error] [--warning] + [--info] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] ARCHIVE @@ -16,10 +17,12 @@ borg debug-dump-archive-items optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc index f3213152a..ef5b78d38 100644 --- a/docs/usage/debug-get-obj.rst.inc +++ b/docs/usage/debug-get-obj.rst.inc @@ -4,7 +4,8 @@ borg debug-get-obj ------------------ :: - usage: borg debug-get-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg debug-get-obj [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] ID PATH @@ -17,10 +18,12 @@ borg debug-get-obj optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc index 44767c271..32a4558da 100644 --- a/docs/usage/debug-put-obj.rst.inc +++ b/docs/usage/debug-put-obj.rst.inc @@ -4,7 +4,8 @@ borg debug-put-obj ------------------ :: - usage: borg debug-put-obj [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg debug-put-obj [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] PATH [PATH ...] @@ -16,10 +17,12 @@ borg debug-put-obj optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index a278cc92e..8e453ab28 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -4,9 +4,9 @@ borg delete ----------- :: - usage: borg delete [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-p] - [-s] [-c] [--save-space] + usage: borg delete [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-p] [-s] [-c] [--save-space] [TARGET] Delete an existing repository or archive @@ -16,10 +16,12 @@ borg delete optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index 8c3a3b9cf..227b254e9 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -4,11 +4,11 @@ borg extract ------------ :: - usage: borg extract [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [--list] [-n] [-e PATTERN] [--exclude-from EXCLUDEFILE] - [--numeric-owner] [--strip-components NUMBER] [--stdout] - [--sparse] + usage: borg extract [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--list] [-n] [-e PATTERN] + [--exclude-from EXCLUDEFILE] [--numeric-owner] + [--strip-components NUMBER] [--stdout] [--sparse] ARCHIVE [PATH [PATH ...]] Extract archive contents @@ -19,10 +19,12 @@ borg extract optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index 7c2c44b5d..1c57c5b56 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -4,8 +4,9 @@ borg info --------- :: - usage: borg info [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] + usage: borg info [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] ARCHIVE Show archive details such as disk space used @@ -15,10 +16,12 @@ borg info optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index 0f37444f2..9af244271 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -4,9 +4,9 @@ borg init --------- :: - usage: borg init [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [-e {none,keyfile,repokey}] + usage: borg init [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-e {none,keyfile,repokey}] [REPOSITORY] Initialize an empty repository @@ -16,10 +16,12 @@ borg init optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 33593017f..4725db5e0 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -4,9 +4,10 @@ borg list --------- :: - usage: borg list [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [--short] - [--list-format LISTFORMAT] [-P PREFIX] + usage: borg list [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--short] [--list-format LISTFORMAT] + [-P PREFIX] [REPOSITORY_OR_ARCHIVE] List archive or repository contents @@ -17,10 +18,12 @@ borg list optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/migrate-to-repokey.rst.inc b/docs/usage/migrate-to-repokey.rst.inc index 2f662c70a..674c0f774 100644 --- a/docs/usage/migrate-to-repokey.rst.inc +++ b/docs/usage/migrate-to-repokey.rst.inc @@ -4,7 +4,8 @@ borg migrate-to-repokey ----------------------- :: - usage: borg migrate-to-repokey [-h] [-v] [--debug] [--lock-wait N] [--show-rc] + usage: borg migrate-to-repokey [-h] [--critical] [--error] [--warning] + [--info] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] [--remote-path PATH] [REPOSITORY] @@ -16,10 +17,12 @@ borg migrate-to-repokey optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index 380df549b..b41886aab 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -4,9 +4,9 @@ borg mount ---------- :: - usage: borg mount [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-f] - [-o OPTIONS] + usage: borg mount [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-f] [-o OPTIONS] REPOSITORY_OR_ARCHIVE MOUNTPOINT Mount archive or an entire repository as a FUSE fileystem @@ -18,10 +18,12 @@ borg mount optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index 06718004b..3dc67b7fa 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -4,11 +4,11 @@ borg prune ---------- :: - usage: borg prune [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-n] - [-s] [--list] [--keep-within WITHIN] [-H HOURLY] [-d DAILY] - [-w WEEKLY] [-m MONTHLY] [-y YEARLY] [-P PREFIX] - [--save-space] + usage: borg prune [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-n] [-s] [--list] + [--keep-within WITHIN] [-H HOURLY] [-d DAILY] [-w WEEKLY] + [-m MONTHLY] [-y YEARLY] [-P PREFIX] [--save-space] [REPOSITORY] Prune repository archives according to specified rules @@ -18,10 +18,12 @@ borg prune optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc index 8e0a4b617..483aa2516 100644 --- a/docs/usage/rename.rst.inc +++ b/docs/usage/rename.rst.inc @@ -4,8 +4,9 @@ borg rename ----------- :: - usage: borg rename [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] + usage: borg rename [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] ARCHIVE NEWNAME Rename an existing archive @@ -16,10 +17,12 @@ borg rename optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index 1e29ff2a4..70a0c5866 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -4,19 +4,21 @@ borg serve ---------- :: - usage: borg serve [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [--restrict-to-path PATH] + usage: borg serve [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [--restrict-to-path PATH] Start in server mode. This command is usually not used manually. optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index 891f645e8..bd920149b 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -4,9 +4,9 @@ borg upgrade ------------ :: - usage: borg upgrade [-h] [-v] [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] [-p] - [-n] [-i] + usage: borg upgrade [-h] [--critical] [--error] [--warning] [--info] [--debug] + [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] + [--remote-path PATH] [-p] [-n] [-i] [REPOSITORY] upgrade a repository from a previous version @@ -16,10 +16,12 @@ borg upgrade optional arguments: -h, --help show this help message and exit - -v, --verbose, --info - enable informative (verbose) output, work on log level - INFO - --debug enable debug output, work on log level DEBUG + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG --lock-wait N wait for the lock, but max. N seconds (default: 1). --show-rc show/log the return code (rc) --no-files-cache do not load/update the file metadata cache used to From 6a3f2d78644567f6b105414916b449eaa4409540 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 8 Apr 2016 23:41:15 +0200 Subject: [PATCH 13/13] update CHANGES --- docs/changes.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index a4cc750e3..5589ea41c 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,8 +1,8 @@ Changelog ========= -Version 1.0.1 (not released yet) --------------------------------- +Version 1.0.1 +------------- New features: @@ -43,6 +43,7 @@ Other changes: - update llfuse requirement, llfuse 1.0 works - update OS / dist packages on build machines, #717 +- prefer showing --info over -v in usage help, #859 - docs: - fix cygwin requirements (gcc-g++)