From 74ab7f1f612cbbaf040847a56b0059df651e3ba5 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Thu, 10 Nov 2016 16:48:35 +0100 Subject: [PATCH 01/10] .gitattributes: Set python diff mode for all *.py files. --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index a97e72971..9d00a6907 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ borg/_version.py export-subst + +*.py diff=python From 5fecac63a929a710f9b6eb17b81a5b7e7911c64b Mon Sep 17 00:00:00 2001 From: enkore Date: Thu, 10 Nov 2016 17:35:59 +0100 Subject: [PATCH 02/10] testsuite/archiver: fix missing newline before RemoteArchiverTestCase --- borg/testsuite/archiver.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index b0fa1a118..bc49bc66b 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1444,6 +1444,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): self.cmd('check', self.repository_location, exit_code=0) self.cmd('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0) + @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs') class RemoteArchiverTestCase(ArchiverTestCase): prefix = '__testsuite__:' From 3ee019761aff9deecef8096b3eb2a1a57bf02a8a Mon Sep 17 00:00:00 2001 From: Johannes Wienke Date: Thu, 10 Nov 2016 19:40:08 +0100 Subject: [PATCH 03/10] Clarify prune behavior for different archive contents In the online help, explain that archives with different contents need to be separated via the prefix when pruning to achieve a desired retention policy per archive set. Relates to #1824. --- borg/archiver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index f417c245f..07086709b 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1536,6 +1536,8 @@ class Archiver: considered for deletion and only those archives count towards the totals specified by the rules. Otherwise, *all* archives in the repository are candidates for deletion! + There is no automatic distinction between archives representing different + contents. These need to be distinguished by specifying matching prefixes. """) subparser = subparsers.add_parser('prune', parents=[common_parser], description=self.do_prune.__doc__, From cd8dfda31873bc3528a43f82e932f5bcfa77120f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Nov 2016 22:55:05 +0100 Subject: [PATCH 04/10] caskroom osxfuse-beta gone, it's osxfuse now (3.5.3) --- .travis/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis/install.sh b/.travis/install.sh index 64ccd5a24..92f14e21c 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -17,7 +17,7 @@ if [[ "$(uname -s)" == 'Darwin' ]]; then brew install lz4 brew outdated pyenv || brew upgrade pyenv brew install pkg-config - brew install Caskroom/versions/osxfuse-beta + brew install Caskroom/versions/osxfuse case "${TOXENV}" in py34) From 3237db46213bd4e6e93144580fa826df9cc73f15 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 12 Nov 2016 23:10:40 +0100 Subject: [PATCH 05/10] at xattr module import time, loggers are not initialized yet --- borg/xattr.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/borg/xattr.py b/borg/xattr.py index 10f99ae66..50d1b0b81 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -12,9 +12,6 @@ from distutils.version import LooseVersion from .helpers import Buffer -from .logger import create_logger -logger = create_logger() - try: ENOATTR = errno.ENOATTR @@ -68,7 +65,7 @@ if libc_name is None: libc_name = 'libc.dylib' else: msg = "Can't find C library. No fallback known. Try installing ldconfig, gcc/cc or objdump." - logger.error(msg) + print(msg, file=sys.stderr) # logger isn't initialized at this stage raise Exception(msg) # If we are running with fakeroot on Linux, then use the xattr functions of fakeroot. This is needed by From 960c42193acdab214217dd8a472f7f28d7ca5153 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 11:19:58 +0100 Subject: [PATCH 06/10] fix tox build for environment-python != containing-python in yet-another instance this instance: the repository worktree is *not* named borg. Cherry pick of 4f1157c into 1.0-maint due to f3efcdb TODO removed since we already did that after 1.0-maint, but 1.0-maint will never receive the change. --- conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/conftest.py b/conftest.py index d80aed4c1..d5423660b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,5 @@ import os +import os.path import sys import pytest @@ -10,12 +11,10 @@ import pytest # The workaround is to remove entries pointing there from the path and check whether "borg" # is still importable. If it is not, then it has not been installed in the environment # and the entries are put back. -# -# TODO: After moving the package to src/: remove this. original_path = list(sys.path) for entry in original_path: - if entry == '' or entry.endswith('/borg'): + if entry == '' or entry == os.path.dirname(__file__): sys.path.remove(entry) try: From 64a3fa8e737a35f3044e8bf654e932fe0bd84d96 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 11:40:19 +0100 Subject: [PATCH 07/10] check: bail out early if repository is *completely* empty --- borg/repository.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/borg/repository.py b/borg/repository.py index fa6458c61..690e77707 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -438,6 +438,9 @@ class Repository: transaction_id = self.get_index_transaction_id() if transaction_id is None: transaction_id = self.io.get_latest_segment() + if transaction_id is None: + report_error('This repository contains no valid data.') + return False if repair: self.io.cleanup(transaction_id) segments_transaction_id = self.io.get_segments_transaction_id() From 2261709e78b34330ed439a64001cc1bb661828cf Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 11:45:35 +0100 Subject: [PATCH 08/10] check: handle repo w/o objects gracefully normal check would complete, --repair would crash when trying to write the rebuilt (empty) manifest out, since self.key was None --- borg/archive.py | 3 +++ borg/testsuite/archiver.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/borg/archive.py b/borg/archive.py index e725857e5..654a1da9a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -833,6 +833,9 @@ class ArchiveChecker: self.repair = repair self.repository = repository self.init_chunks() + if not self.chunks: + logger.error('Repository contains no apparent data at all, cannot continue check/repair.') + return False self.key = self.identify_key(repository) if Manifest.MANIFEST_ID not in self.chunks: logger.error("Repository manifest not found!") diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index bc49bc66b..b50304b5a 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1444,6 +1444,13 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): self.cmd('check', self.repository_location, exit_code=0) self.cmd('extract', '--dry-run', self.repository_location + '::archive1', exit_code=0) + def test_empty_repository(self): + with Repository(self.repository_location, exclusive=True) as repository: + for id_ in repository.list(): + repository.delete(id_) + repository.commit() + self.cmd('check', self.repository_location, exit_code=1) + @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs') class RemoteArchiverTestCase(ArchiverTestCase): From 639eba16354008ac7d97d966084eac2fb930a262 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 15:22:51 +0100 Subject: [PATCH 09/10] Fix check incorrectly reporting attic 0.13 and earlier archives as corrupt --- borg/archive.py | 3 +++ borg/testsuite/archiver.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/borg/archive.py b/borg/archive.py index 654a1da9a..b2f9df70f 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1020,6 +1020,9 @@ class ArchiveChecker: def valid_item(obj): if not isinstance(obj, StableDict): return False + # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item. + # We ignore it here, should it exist. See test_attic013_acl_bug for details. + obj.pop(b'acl', None) keys = set(obj) return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index b50304b5a..a7d8ff3e6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1451,6 +1451,25 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): repository.commit() self.cmd('check', self.repository_location, exit_code=1) + def test_attic013_acl_bug(self): + # Attic up to release 0.13 contained a bug where every item unintentionally received + # a b'acl'=None key-value pair. + # This bug can still live on in Borg repositories (through borg upgrade). + archive, repository = self.open_archive('archive1') + with repository: + manifest, key = Manifest.load(repository) + with Cache(repository, key, manifest) as cache: + archive = Archive(repository, key, manifest, '0.13', cache=cache, create=True) + archive.items_buffer.add({ + # path and mtime are required. + b'path': '1234', + b'mtime': 0, + # acl is the offending key. + b'acl': None + }) + archive.save() + self.cmd('check', self.repository_location, exit_code=0) + @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs') class RemoteArchiverTestCase(ArchiverTestCase): From a898297669213fb95eab4f71a838121993775e53 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 02:00:39 +0100 Subject: [PATCH 10/10] check: improve "did not get expected metadata dict" diagnostic --- borg/archive.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index b2f9df70f..da8a0e70a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1017,14 +1017,21 @@ class ArchiveChecker: self.error_found = True logger.error(msg) + def list_keys_safe(keys): + return ', '.join((k.decode() if isinstance(k, bytes) else str(k) for k in keys)) + def valid_item(obj): if not isinstance(obj, StableDict): - return False + return False, 'not a dictionary' # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item. # We ignore it here, should it exist. See test_attic013_acl_bug for details. obj.pop(b'acl', None) keys = set(obj) - return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys) + if not REQUIRED_ITEM_KEYS.issubset(keys): + return False, 'missing required keys: ' + list_keys_safe(REQUIRED_ITEM_KEYS - keys) + if not keys.issubset(item_keys): + return False, 'invalid keys: ' + list_keys_safe(keys - item_keys) + return True, '' i = 0 for state, items in groupby(archive[b'items'], missing_chunk_detector): @@ -1040,10 +1047,11 @@ class ArchiveChecker: unpacker.feed(self.key.decrypt(chunk_id, cdata)) try: for item in unpacker: - if valid_item(item): + valid, reason = valid_item(item) + if valid: yield item else: - report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i) + report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i) except RobustUnpacker.UnpackerCrashed as err: report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i) unpacker.resync()