From 33251269885f36c37f325831afbc32b44284e549 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 4 Jul 2016 15:06:20 +0200 Subject: [PATCH 01/15] add missing BORG_REMOTE_PATH env var, fixes #1258 --- borg/archiver.py | 4 ++-- borg/remote.py | 3 ++- docs/usage.rst | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 49e0bdbaa..0d2662d00 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -912,8 +912,8 @@ class Archiver: help='do not load/update the file metadata cache used to detect unchanged files') common_parser.add_argument('--umask', dest='umask', type=lambda s: int(s, 8), default=UMASK_DEFAULT, metavar='M', help='set umask to M (local and remote, default: %(default)04o)') - common_parser.add_argument('--remote-path', dest='remote_path', default='borg', metavar='PATH', - help='set remote path to executable (default: "%(default)s")') + common_parser.add_argument('--remote-path', dest='remote_path', metavar='PATH', + help='set remote path to executable (default: "borg")') parser = argparse.ArgumentParser(prog=prog, description='Borg - Deduplicated Backups') parser.add_argument('-V', '--version', action='version', version='%(prog)s ' + __version__, diff --git a/borg/remote.py b/borg/remote.py index 26dce53d4..a8e64c127 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -220,7 +220,8 @@ class RemoteRepository: if testing: return [sys.executable, '-m', 'borg.archiver', 'serve'] + opts + self.extra_test_args else: # pragma: no cover - return [args.remote_path, 'serve'] + opts + remote_path = args.remote_path or os.environ.get('BORG_REMOTE_PATH', 'borg') + return [remote_path, 'serve'] + opts def ssh_cmd(self, location): """return a ssh command line that can be prefixed to a borg command line""" diff --git a/docs/usage.rst b/docs/usage.rst index ca8d536db..8b5dd50b3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -82,6 +82,9 @@ General: BORG_RSH When set, use this command instead of ``ssh``. This can be used to specify ssh options, such as a custom identity file ``ssh -i /path/to/private/key``. See ``man ssh`` for other options. + BORG_REMOTE_PATH + When set, use the given path/filename as remote path (default is "borg"). + Using ``--remote-path PATH`` commandline option overrides the environment variable. TMPDIR where temporary files are stored (might need a lot of temporary space for some operations) From 0b488760a7e5812f18d066b48c84d69801cb2912 Mon Sep 17 00:00:00 2001 From: Michael Gajda Date: Mon, 4 Jul 2016 21:51:11 +0200 Subject: [PATCH 02/15] Update docs about lzma compreesion levels above 6 lzma compression levels above 6 do not increase compression ration but waste a lot of cpu cycles. --- docs/usage/create.rst.inc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index 74c228b6c..9d536274f 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -70,7 +70,8 @@ borg create compression (default), lz4 == lz4, zlib == zlib (default level 6), zlib,0 .. zlib,9 == zlib (with level 0..9), lzma == lzma (default level 6), lzma,0 .. - lzma,9 == lzma (with level 0..9). + lzma,6 == lzma (with level 0..6. levels above 6 do not + improve compression and only waste many cpu cycles) --read-special open and read special files as if they were regular files -n, --dry-run do not create a backup archive From 20eab2b5c8d3ea63a2d6396ad77461b86bffeee3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 4 Jul 2016 22:37:24 +0200 Subject: [PATCH 03/15] update frontpage / support docs less items in frontpage link list add IRC chat link minor other changes --- README.rst | 15 ++++++++------- docs/support.rst | 6 +++--- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/README.rst b/README.rst index 2417b24d5..a9d338151 100644 --- a/README.rst +++ b/README.rst @@ -113,13 +113,14 @@ Links ===== * `Main Web Site `_ -* `Releases `_ -* `PyPI packages `_ -* `ChangeLog `_ -* `GitHub `_ -* `Issue Tracker `_ -* `Bounties & Fundraisers `_ -* `Mailing List `_ +* `Releases `_, + `PyPI packages `_ and + `ChangeLog `_ +* `GitHub `_, + `Issue Tracker `_ and + `Bounties & Fundraisers `_ +* `Web-Chat (IRC) `_ and + `Mailing List `_ * `License `_ Notes diff --git a/docs/support.rst b/docs/support.rst index 7cd1890d3..9d64621fc 100644 --- a/docs/support.rst +++ b/docs/support.rst @@ -16,15 +16,15 @@ ticket on the project's `issue tracker`_. For more general questions or discussions, IRC or mailing list are preferred. -IRC ---- +Chat (IRC) +---------- Join us on channel #borgbackup on chat.freenode.net. As usual on IRC, just ask or tell directly and then patiently wait for replies. Stay connected. You could use the following link (after connecting, you can change the random -nickname you got by typing "/nick mydesirednickname"): +nickname you get by typing "/nick mydesirednickname"): http://webchat.freenode.net/?randomnick=1&channels=%23borgbackup&uio=MTY9dHJ1ZSY5PXRydWUa8 From 07d0a61e4601343025548bf8a6d3193fc8e4e3fa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 1 Jul 2016 04:27:06 +0200 Subject: [PATCH 04/15] forced archive deletion, fixes #1139 --- borg/archive.py | 68 +++++++++++++++++++++++++++++++++++++----------- borg/archiver.py | 10 +++++-- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 1ce93ab5d..fad821a0b 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -24,6 +24,8 @@ from .helpers import Error, uid2user, user2uid, gid2group, group2gid, \ from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex +from .repository import Repository + import msgpack ITEMS_BUFFER = 1024 * 1024 @@ -494,23 +496,59 @@ Number of files: {0.stats.nfiles}'''.format( self.cache.chunk_decref(self.id, self.stats) del self.manifest.archives[self.name] - def delete(self, stats, progress=False): - unpacker = msgpack.Unpacker(use_list=False) - items_ids = self.metadata[b'items'] - pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) - for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + def delete(self, stats, progress=False, forced=False): + class ChunksIndexError(Error): + """Chunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.""" + + def chunk_decref(id, stats): + nonlocal error + try: + self.cache.chunk_decref(id, stats) + except KeyError: + cid = hexlify(id).decode('ascii') + raise ChunksIndexError(cid) + except Repository.ObjectNotFound as e: + # object not in repo - strange, but we wanted to delete it anyway. + if not forced: + raise + error = True + + error = False + try: + unpacker = msgpack.Unpacker(use_list=False) + items_ids = self.metadata[b'items'] + pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) + for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + if progress: + pi.show(i) + unpacker.feed(self.key.decrypt(items_id, data)) + chunk_decref(items_id, stats) + try: + for item in unpacker: + if b'chunks' in item: + for chunk_id, size, csize in item[b'chunks']: + chunk_decref(chunk_id, stats) + except (TypeError, ValueError): + # if items metadata spans multiple chunks and one chunk got dropped somehow, + # it could be that unpacker yields bad types + if not forced: + raise + error = True if progress: - pi.show(i) - unpacker.feed(self.key.decrypt(items_id, data)) - self.cache.chunk_decref(items_id, stats) - for item in unpacker: - if b'chunks' in item: - for chunk_id, size, csize in item[b'chunks']: - self.cache.chunk_decref(chunk_id, stats) - if progress: - pi.finish() - self.cache.chunk_decref(self.id, stats) + pi.finish() + except (msgpack.UnpackException, Repository.ObjectNotFound): + # items metadata corrupted + if not forced: + raise + error = True + # in forced delete mode, we try hard to delete at least the manifest entry, + # if possible also the archive superblock, even if processing the items raises + # some harmless exception. + chunk_decref(self.id, stats) del self.manifest.archives[self.name] + if error: + logger.warning('forced deletion succeeded, but the deleted archive was corrupted.') + logger.warning('borg check --repair is required to free all space.') def stat_attrs(self, st, path): item = { diff --git a/borg/archiver.py b/borg/archiver.py index 49e0bdbaa..e5f1af91d 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -426,7 +426,7 @@ class Archiver: with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() - archive.delete(stats, progress=args.progress) + archive.delete(stats, progress=args.progress, forced=args.forced) manifest.write() repository.commit(save_space=args.save_space) cache.commit() @@ -635,7 +635,7 @@ class Archiver: else: if args.output_list: logger.info('Pruning archive: %s' % format_archive(archive)) - Archive(repository, key, manifest, archive.name, cache).delete(stats) + Archive(repository, key, manifest, archive.name, cache).delete(stats, forced=args.forced) if to_delete and not args.dry_run: manifest.write() repository.commit(save_space=args.save_space) @@ -1230,6 +1230,9 @@ class Archiver: subparser.add_argument('-c', '--cache-only', dest='cache_only', action='store_true', default=False, help='delete only the local cache for the given repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force deletion of corrupted archives') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') @@ -1342,6 +1345,9 @@ class Archiver: subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not change repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force pruning of corrupted archives') subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the deleted archive') From 54e566747745baf6a94ec8d7d144e0f20b0b52db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Jun 2016 02:33:49 +0200 Subject: [PATCH 05/15] update CHANGES --- docs/changes.rst | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 23ff0bdad..35c22d321 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,94 @@ Changelog ========= +Version 1.0.4 (not released yet) +-------------------------------- + +New features: + +- borg serve --append-only, #1168 + This was included because it was a simple change (append-only functionality + was already present via repository config file) and makes better security now + practically usable. +- BORG_REMOTE_PATH environment variable, #1258 + This was included because it was a simple change (--remote-path cli option + was already present) and makes borg much easier to use if you need it. + +Bug fixes: + +- Fix wrong handling and reporting of OSErrors in borg create, #1138. + This was a serious issue: in the context of "borg create", errors like + repository I/O errors (e.g. disk I/O errors, ssh repo connection errors) + were handled badly and did not lead to a crash (which would be good for this + case, because the repo transaction would be incomplete and trigger a + transaction rollback to clean up). + Now, error handling for source files is cleanly separated from every other + error handling, so only problematic input files are logged and skipped. +- Implement fail-safe error handling for borg extract. + Note that this isn't nearly as critical as the borg create error handling + bug, since nothing is written to the repo. So this was "merely" misleading + error reporting. +- Add missing error handler in directory attr restore loop. +- repo: make sure write data hits disk before the commit tag (#1236) and also + sync the containing directory. +- fixes for --read-special mode: + + - ignore known files cache, #1241 + - fake regular file mode, #1214 + - improve symlinks handling, #1215 +- remove passphrase from subprocess environment, #1105 +- Ignore empty index file (will trigger index rebuild), #1195 +- add missing placeholder support for --prefix, #1027 +- improve exception handling for placeholder replacement +- catch and format exceptions in arg parsing +- helpers: fix "undefined name 'e'" in exception handler +- better error handling for missing repo manifest, #1043 +- borg delete: + + - make it possible to delete a repo without manifest + - borg delete --forced allows to delete corrupted archives, #1139 +- borg check: + + - make borg check work for empty repo + - fix resync and msgpacked item qualifier, #1135 + - rebuild_manifest: fix crash if 'name' or 'time' key were missing. + - better validation of item metadata dicts, #1130 + - better validation of archive metadata dicts +- close the repo on exit - even if rollback did not work, #1197. + This is rather cosmetic, it avoids repo closing in the destructor. + +- tests: + + - fix sparse file test, #1170 + - flake8: ignore new F405, #1185 + +Other changes: + +- make borg build/work on OpenSSL 1.0 and 1.1, #1187 +- docs / help: + + - fix / clarify prune help, #1143 + - fix "patterns" help formatting + - add missing docs / help about placeholders + - resources: rename atticmatic to borgmatic + - document sshd settings, #545 + - more details about checkpoints, add split trick, #1171 + - support docs: add freenode web chat link, #1175 + - add prune visualization / example, #723 + - add note that Fnmatch is default, #1247 + - make clear that lzma levels > 6 are a waste of cpu cycles +- repository interoperability with borg master (1.1dev) branch: + + - borg check: read item metadata keys from manifest, #1147 + - read v2 hints files, #1235 + - fix hints file "unknown version" error handling bug +- tests: add tests for format_line +- llfuse: update version requirement for freebsd +- Vagrantfile: use openbsd 5.9, #716 +- use Python 3.5.2 to build the binaries +- add .eggs to .gitignore + + Version 1.0.3 ------------- From 19d1bbfa14ea345b1d27505cee43670eea4e922a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 00:44:00 +0200 Subject: [PATCH 06/15] add advice about issue #1138, fixes #1223 --- docs/changes.rst | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 35c22d321..60813c0b1 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,55 @@ Changelog ========= +Important note about pre-1.0.4 potential repo corruption +-------------------------------------------------------- + +Some external errors (like network or disk I/O errors) could lead to +corruption of the backup repository due to issue #1138. + +A sign that this happened is if "E" status was reported for a file that can +not be explained by problems with the source file. If you still have logs from +"borg create -v --list", you can check for "E" status. + +Here is what could cause corruption and what you can do now: + +1) I/O errors (e.g. repo disk errors) while writing data to repo. + +This could lead to corrupted segment files. + +Fix:: + + # check for corrupt chunks / segments: + borg check -v --repository-only REPO + + # repair the repo: + borg check -v --repository-only --repair REPO + + # make sure everything is fixed: + borg check -v --repository-only REPO + +2) Unreliable network / unreliable connection to the repo. + +This could lead to archive metadata corruption. + +Fix:: + + # check for corrupt archives: + borg check -v --archives-only REPO + + # delete the corrupt archives: + borg delete --force REPO::CORRUPT_ARCHIVE + + # make sure everything is fixed: + borg check -v --archives-only REPO + +3) In case you want to do more intensive checking. + +The best check that everything is ok is to run a dry-run extraction:: + + borg extract -v --dry-run REPO::ARCHIVE + + Version 1.0.4 (not released yet) -------------------------------- From 498d4b141e9f71a7b0a9aa00942894d4697e1e47 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 16:49:50 +0200 Subject: [PATCH 07/15] sparse test: fix sparseness assertion in test prep, fixes #1264 use the same condition for the input file as also later for the extracted file. the test preparation sparseness assertion failed on cygwin / ntfs, because the input file uses ~40MB in blocks vs. total_len ~80MB. --- borg/testsuite/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 126d1c5a5..e8039a611 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -398,7 +398,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): st = os.stat(filename) self.assert_equal(st.st_size, total_len) if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len / 9) # is input sparse? + self.assert_true(st.st_blocks * 512 < total_len) # is input sparse? self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') with changedir('output'): @@ -415,7 +415,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): if sparse_support: if hasattr(st, 'st_blocks'): # do only check if it is less, do NOT check if it is much less - # as that causes troubles on xfs and zfs: + # as that causes troubles on xfs, zfs, ntfs: self.assert_true(st.st_blocks * 512 < total_len) if hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): with open(filename, 'rb') as fd: From 72a7c0bae5f61fb33eeb25f64e5f2c90f1e7189b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 17:20:23 +0200 Subject: [PATCH 08/15] catch "invalid argument" on cygwin, fixes #257 obviously cygwin (windows) does not like the made-up uid 100 gid 200 and raises "invalid argument" OSError. --- borg/testsuite/archiver.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 126d1c5a5..59bbeb2b8 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -288,10 +288,14 @@ class ArchiverTestCase(ArchiverTestCaseBase): # File mode os.chmod('input/dir2', 0o555) # if we take away write perms, we need root to remove contents # File owner - os.chown('input/file1', 100, 200) + os.chown('input/file1', 100, 200) # raises OSError invalid argument on cygwin have_root = True # we have (fake)root except PermissionError: have_root = False + except OSError as e: + if e.errno != errno.EINVAL: + raise + have_root = False return have_root def test_basic_functionality(self): From 2922948100fa060e908216cbdfd973856eed31b6 Mon Sep 17 00:00:00 2001 From: Jens Rantil Date: Tue, 5 Jul 2016 17:28:24 +0200 Subject: [PATCH 09/15] Update faq.rst Fix misspelling. --- docs/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/faq.rst b/docs/faq.rst index 100abd89b..ff0e88ad3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -223,7 +223,7 @@ archive), so it does not need to get transmitted and stored again. Once your backup has finished successfully, you can delete all ``.checkpoint`` archives. -How can I backup huge file(s) over a instable connection? +How can I backup huge file(s) over a unstable connection? --------------------------------------------------------- You can use this "split trick" as a workaround for the in-between-files-only From 7ed425dae0628fc27b57aacef7cd01f0ef9411a1 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 5 Jul 2016 11:58:44 +0200 Subject: [PATCH 10/15] Repository: cleanup on ENOSPC --- borg/repository.py | 8 +++++++- docs/internals.rst | 1 + docs/quickstart.rst | 10 +++++++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/borg/repository.py b/borg/repository.py index 972baab09..87666a573 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -84,6 +84,12 @@ class Repository: def __exit__(self, exc_type, exc_val, exc_tb): if exc_type is not None: + no_space_left_on_device = exc_type is OSError and exc_val.errno == errno.ENOSPC + # The ENOSPC could have originated somewhere else besides the Repository. The cleanup is always safe, unless + # EIO or FS corruption ensues, which is why we specifically check for ENOSPC. + if self._active_txn and no_space_left_on_device: + logger.warning('No space left on device, cleaning up partial transaction to free space.') + self.io.cleanup(self.io.get_segments_transaction_id()) self.rollback() self.close() @@ -543,7 +549,7 @@ class LoggedIO: return None def get_segments_transaction_id(self): - """Verify that the transaction id is consistent with the index transaction id + """Return last committed segment """ for segment, filename in self.segment_iterator(reverse=True): if self.is_committed_segment(segment): diff --git a/docs/internals.rst b/docs/internals.rst index 9d1bbd84c..1bd14bb1e 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -280,6 +280,7 @@ emptied to 25%, its size is shrinked. So operations on it have a variable complexity between constant and linear with low factor, and memory overhead varies between 33% and 300%. +.. _cache-memory-usage: Indexes / Caches memory usage ----------------------------- diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 81d358d4c..31d2e216b 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -11,11 +11,15 @@ The next section continues by showing how backups can be automated. Important note about free space ------------------------------- -Before you start creating backups, please make sure that there is **always** +Before you start creating backups, please make sure that there is *always* a good amount of free space on the filesystem that has your backup repository -(and also on ~/.cache). It is hard to tell how much, maybe 1-5%. +(and also on ~/.cache). A few GB should suffice for most hard-drive sized +repositories. See also :ref:`cache-memory-usage`. -If you run out of disk space, it can be hard or impossible to free space, +If |project_name| runs out of disk space, it tries to free as much space as it +can while aborting the current operation safely, which allows to free more space +by deleting/pruning archives. This mechanism is not bullet-proof though. +If you *really* run out of disk space, it can be hard or impossible to free space, because |project_name| needs free space to operate - even to delete backup archives. There is a ``--save-space`` option for some commands, but even with that |project_name| will need free space to operate. From e023053472e945afd1d175373547a5484d5a4655 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 22:43:38 +0200 Subject: [PATCH 11/15] add a glibc compat check script --- scripts/glibc_check.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 scripts/glibc_check.py diff --git a/scripts/glibc_check.py b/scripts/glibc_check.py new file mode 100644 index 000000000..a400bbd1d --- /dev/null +++ b/scripts/glibc_check.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +""" +Check if all given binaries work with the given glibc version. + +check_glibc.py 2.11 bin [bin ...] +""" + +import re +import subprocess +import sys + +verbose = True +objdump = "objdump -T %s" +glibc_re = re.compile(r'GLIBC_([0-9]\.[0-9]+)') + + +def parse_version(v): + major, minor = v.split('.') + return int(major), int(minor) + + +def format_version(version): + return "%d.%d" % version + + +def main(): + given = parse_version(sys.argv[1]) + filenames = sys.argv[2:] + + overall_versions = set() + for filename in filenames: + try: + output = subprocess.check_output(objdump % filename, shell=True, + stderr=subprocess.STDOUT) + output = output.decode('utf-8') + versions = set(parse_version(match.group(1)) + for match in glibc_re.finditer(output)) + requires_glibc = max(versions) + overall_versions.add(requires_glibc) + if verbose: + print("%s %s" % (filename, format_version(requires_glibc))) + except subprocess.CalledProcessError as e: + if verbose: + print("%s errored." % filename) + + wanted = max(overall_versions) + ok = given >= wanted + + if verbose: + if ok: + print("The binaries work with the given glibc %s." % + format_version(given)) + else: + print("The binaries do not work with the given glibc %s. " + "Minimum is: %s" % (format_version(given), format_version(wanted))) + return ok + + +if __name__ == '__main__': + ok = main() + sys.exit(0 if ok else 1) From e0f549a01d9d1219127decb8dd0c8b7480acef3f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 22:45:25 +0200 Subject: [PATCH 12/15] move the hash sizes script also to scripts/ --- {borg => scripts}/hash_sizes.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {borg => scripts}/hash_sizes.py (100%) diff --git a/borg/hash_sizes.py b/scripts/hash_sizes.py similarity index 100% rename from borg/hash_sizes.py rename to scripts/hash_sizes.py From 2f2925ed89e2455b1d05804f0c0c22522056b4d6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 23:29:23 +0200 Subject: [PATCH 13/15] add a do-not-edit note to auto-generated files --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6312aee09..7aca6be78 100644 --- a/setup.py +++ b/setup.py @@ -163,6 +163,7 @@ class build_usage(Command): for command, parser in choices.items(): print('generating help for %s' % command) with open('docs/usage/%s.rst.inc' % command, 'w') as doc: + doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") if command == 'help': for topic in Archiver.helptext: params = {"topic": topic, From 740c638030a558b922f57f8840ac33f097421707 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Jul 2016 23:30:08 +0200 Subject: [PATCH 14/15] update usage files --- docs/usage/break-lock.rst.inc | 2 + docs/usage/change-passphrase.rst.inc | 2 + docs/usage/check.rst.inc | 2 + docs/usage/create.rst.inc | 10 +-- docs/usage/debug-delete-obj.rst.inc | 2 + docs/usage/debug-dump-archive-items.rst.inc | 2 + docs/usage/debug-get-obj.rst.inc | 2 + docs/usage/debug-put-obj.rst.inc | 2 + docs/usage/delete.rst.inc | 6 +- docs/usage/extract.rst.inc | 2 + docs/usage/help.rst.inc | 76 +++++++++++---------- docs/usage/info.rst.inc | 2 + docs/usage/init.rst.inc | 2 + docs/usage/list.rst.inc | 2 + docs/usage/migrate-to-repokey.rst.inc | 2 + docs/usage/mount.rst.inc | 2 + docs/usage/prune.rst.inc | 5 +- docs/usage/rename.rst.inc | 2 + docs/usage/serve.rst.inc | 2 + docs/usage/upgrade.rst.inc | 2 + 20 files changed, 86 insertions(+), 43 deletions(-) diff --git a/docs/usage/break-lock.rst.inc b/docs/usage/break-lock.rst.inc index 4edef5c23..43707040a 100644 --- a/docs/usage/break-lock.rst.inc +++ b/docs/usage/break-lock.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_break-lock: borg break-lock diff --git a/docs/usage/change-passphrase.rst.inc b/docs/usage/change-passphrase.rst.inc index 08ba4ee9f..ac0019285 100644 --- a/docs/usage/change-passphrase.rst.inc +++ b/docs/usage/change-passphrase.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_change-passphrase: borg change-passphrase diff --git a/docs/usage/check.rst.inc b/docs/usage/check.rst.inc index 9ad6a4b35..738f7d614 100644 --- a/docs/usage/check.rst.inc +++ b/docs/usage/check.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_check: borg check diff --git a/docs/usage/create.rst.inc b/docs/usage/create.rst.inc index d294197b6..fbf531bc6 100644 --- a/docs/usage/create.rst.inc +++ b/docs/usage/create.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_create: borg create @@ -70,10 +72,10 @@ borg create compression (default), lz4 == lz4, zlib == zlib (default level 6), zlib,0 .. zlib,9 == zlib (with level 0..9), lzma == lzma (default level 6), lzma,0 .. - lzma,6 == lzma (with level 0..6. levels above 6 do not - improve compression and only waste many cpu cycles) - --read-special open and read special files as if they were regular - files + lzma,9 == lzma (with level 0..9). + --read-special open and read block and char device files as well as + FIFOs as if they were regular files. Also follows + symlinks pointing to these kinds of files. -n, --dry-run do not create a backup archive Description diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc index bfc31043a..7ed144bfb 100644 --- a/docs/usage/debug-delete-obj.rst.inc +++ b/docs/usage/debug-delete-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-delete-obj: borg debug-delete-obj diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc index fd2cd62c9..e83bbf854 100644 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ b/docs/usage/debug-dump-archive-items.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-dump-archive-items: borg debug-dump-archive-items diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc index ef5b78d38..f38f0e884 100644 --- a/docs/usage/debug-get-obj.rst.inc +++ b/docs/usage/debug-get-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-get-obj: borg debug-get-obj diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc index 32a4558da..4f02324bf 100644 --- a/docs/usage/debug-put-obj.rst.inc +++ b/docs/usage/debug-put-obj.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_debug-put-obj: borg debug-put-obj diff --git a/docs/usage/delete.rst.inc b/docs/usage/delete.rst.inc index 8e453ab28..64c85bdb9 100644 --- a/docs/usage/delete.rst.inc +++ b/docs/usage/delete.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_delete: borg delete @@ -6,7 +8,8 @@ borg delete usage: borg delete [-h] [--critical] [--error] [--warning] [--info] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-p] [-s] [-c] [--save-space] + [--remote-path PATH] [-p] [-s] [-c] [--force] + [--save-space] [TARGET] Delete an existing repository or archive @@ -31,6 +34,7 @@ borg delete -p, --progress show progress display while deleting a single archive -s, --stats print statistics for the deleted archive -c, --cache-only delete only the local cache for the given repository + --force force deletion of corrupted archives --save-space work slower, but using less space Description diff --git a/docs/usage/extract.rst.inc b/docs/usage/extract.rst.inc index 227b254e9..9cf2d4f9c 100644 --- a/docs/usage/extract.rst.inc +++ b/docs/usage/extract.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_extract: borg extract diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 9b8d23d66..c83dfb0aa 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -1,41 +1,5 @@ -.. _borg_placeholders: +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! -borg help placeholders -~~~~~~~~~~~~~~~~~~~~~~ -:: - - -Repository (or Archive) URLs and --prefix values support these placeholders: - -{hostname} - - The (short) hostname of the machine. - -{fqdn} - - The full name of the machine. - -{now} - - The current local date and time. - -{utcnow} - - The current UTC date and time. - -{user} - - The user name (or UID, if no name is available) of the user running borg. - -{pid} - - The current process ID. - -Examples:: - - borg create /path/to/repo::{hostname}-{user}-{utcnow} ... - borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... .. _borg_patterns: borg help patterns @@ -129,3 +93,41 @@ Examples:: sh:/home/*/.thumbnails EOF $ borg create --exclude-from exclude.txt backup / +.. _borg_placeholders: + +borg help placeholders +~~~~~~~~~~~~~~~~~~~~~~ +:: + + +Repository (or Archive) URLs and --prefix values support these placeholders: + +{hostname} + + The (short) hostname of the machine. + +{fqdn} + + The full name of the machine. + +{now} + + The current local date and time. + +{utcnow} + + The current UTC date and time. + +{user} + + The user name (or UID, if no name is available) of the user running borg. + +{pid} + + The current process ID. + +Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index 1c57c5b56..68c9194ec 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_info: borg info diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index 9af244271..798fe6e53 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_init: borg init diff --git a/docs/usage/list.rst.inc b/docs/usage/list.rst.inc index 4725db5e0..8f1d2a45e 100644 --- a/docs/usage/list.rst.inc +++ b/docs/usage/list.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_list: borg list diff --git a/docs/usage/migrate-to-repokey.rst.inc b/docs/usage/migrate-to-repokey.rst.inc index 674c0f774..c5ff21be3 100644 --- a/docs/usage/migrate-to-repokey.rst.inc +++ b/docs/usage/migrate-to-repokey.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_migrate-to-repokey: borg migrate-to-repokey diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index b41886aab..1754b0e2c 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_mount: borg mount diff --git a/docs/usage/prune.rst.inc b/docs/usage/prune.rst.inc index 08f54605f..274d60c2a 100644 --- a/docs/usage/prune.rst.inc +++ b/docs/usage/prune.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_prune: borg prune @@ -6,7 +8,7 @@ borg prune usage: borg prune [-h] [--critical] [--error] [--warning] [--info] [--debug] [--lock-wait N] [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] [-n] [-s] [--list] + [--remote-path PATH] [-n] [--force] [-s] [--list] [--keep-within WITHIN] [-H HOURLY] [-d DAILY] [-w WEEKLY] [-m MONTHLY] [-y YEARLY] [-P PREFIX] [--save-space] [REPOSITORY] @@ -31,6 +33,7 @@ borg prune --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") -n, --dry-run do not change repository + --force force pruning of corrupted archives -s, --stats print statistics for the deleted archive --list output verbose list of archives it keeps/prunes --keep-within WITHIN keep all archives within this time interval diff --git a/docs/usage/rename.rst.inc b/docs/usage/rename.rst.inc index 483aa2516..cfafb0022 100644 --- a/docs/usage/rename.rst.inc +++ b/docs/usage/rename.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_rename: borg rename diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index e8eb6c028..eb57a975b 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_serve: borg serve diff --git a/docs/usage/upgrade.rst.inc b/docs/usage/upgrade.rst.inc index bd920149b..6c44edf74 100644 --- a/docs/usage/upgrade.rst.inc +++ b/docs/usage/upgrade.rst.inc @@ -1,3 +1,5 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + .. _borg_upgrade: borg upgrade From 8db670cf7fee04e9c79b8a4291bd488d2195bf48 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 6 Jul 2016 00:37:01 +0200 Subject: [PATCH 15/15] update CHANGES --- docs/changes.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 60813c0b1..0dcc2074a 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -62,6 +62,9 @@ New features: - BORG_REMOTE_PATH environment variable, #1258 This was included because it was a simple change (--remote-path cli option was already present) and makes borg much easier to use if you need it. +- Repository: cleanup incomplete transaction on "no space left" condition. + In many cases, this can avoid a 100% full repo filesystem (which is very + problematic as borg always needs free space - even to delete archives). Bug fixes: @@ -110,6 +113,8 @@ Bug fixes: - fix sparse file test, #1170 - flake8: ignore new F405, #1185 + - catch "invalid argument" on cygwin, #257 + - fix sparseness assertion in test prep, #1264 Other changes: @@ -126,6 +131,7 @@ Other changes: - add prune visualization / example, #723 - add note that Fnmatch is default, #1247 - make clear that lzma levels > 6 are a waste of cpu cycles + - add a "do not edit" note to auto-generated files, #1250 - repository interoperability with borg master (1.1dev) branch: - borg check: read item metadata keys from manifest, #1147 @@ -135,6 +141,7 @@ Other changes: - llfuse: update version requirement for freebsd - Vagrantfile: use openbsd 5.9, #716 - use Python 3.5.2 to build the binaries +- glibc compatibility checker: scripts/glibc_check.py - add .eggs to .gitignore