From d2b7dbc0a8c3a95baac2734df375e156caedadd3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 22:51:59 +0100 Subject: [PATCH 1/6] debugging helper: borg debug-dump-archive-items --- borg/archiver.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index e18ded6c6..145ca1e68 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -500,6 +500,20 @@ class Archiver: print("warning: %s" % e) return self.exit_code + def do_debug_dump_archive_items(self, args): + """dump (decrypted, decompressed) archive items metadata (not: data)""" + repository = self.open_repository(args.archive) + manifest, key = Manifest.load(repository) + archive = Archive(repository, key, manifest, args.archive.archive) + for i, item_id in enumerate(archive.metadata[b'items']): + data = key.decrypt(item_id, repository.get(item_id)) + filename = '%06d_%s.items' %(i, hexlify(item_id).decode('ascii')) + print('Dumping', filename) + with open(filename, 'wb') as fd: + fd.write(data) + print('Done.') + return EXIT_SUCCESS + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -985,6 +999,18 @@ class Archiver: subparser.set_defaults(func=functools.partial(self.do_help, parser, subparsers.choices)) subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') + + debug_dump_archive_items_epilog = textwrap.dedent(""" + This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. + """) + subparser = subparsers.add_parser('debug-dump-archive-items', parents=[common_parser], + description=self.do_debug_dump_archive_items.__doc__, + epilog=debug_dump_archive_items_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_dump_archive_items) + subparser.add_argument('archive', metavar='ARCHIVE', + type=location_validator(archive=True), + help='archive to dump') return parser def parse_args(self, args=None): From 47813f6f6a7d747a9b992e697b712ce90b01e11c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 3 Nov 2015 23:45:49 +0100 Subject: [PATCH 2/6] archiver checker: better error logging, give chunk_id and sequence numbers can be used together with borg debug-dump-archive-items. --- borg/archive.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 2333a1029..8e850873a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -788,21 +788,33 @@ class ArchiveChecker: _state += 1 return _state + def report(msg, chunk_id, chunk_no): + cid = hexlify(chunk_id).decode('ascii') + msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items + self.report_progress(msg, error=True) + + i = 0 for state, items in groupby(archive[b'items'], missing_chunk_detector): items = list(items) if state % 2: - self.report_progress('Archive metadata damage detected', error=True) + for chunk_id in items: + report('item metadata chunk missing', chunk_id, i) + i += 1 continue if state > 0: unpacker.resync() for chunk_id, cdata in zip(items, repository.get_many(items)): unpacker.feed(self.key.decrypt(chunk_id, cdata)) - for item in unpacker: - if not isinstance(item, dict): - self.report_progress('Did not get expected metadata dict - archive corrupted!', - error=True) - continue - yield item + try: + for item in unpacker: + if isinstance(item, dict): + yield item + else: + report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i) + except Exception: + report('Exception while unpacking item metadata', chunk_id, i) + raise + i += 1 repository = cache_if_remote(self.repository) if archive is None: From 39b5734b31cfc2733891ca20938a2e86f8edeab6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 4 Nov 2015 01:05:21 +0100 Subject: [PATCH 3/6] debugging helper: borg debug-delete-obj --- borg/archiver.py | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 145ca1e68..9ad291aa1 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -17,7 +17,7 @@ import traceback from . import __version__ from .helpers import Error, location_validator, format_time, format_file_size, \ format_file_mode, ExcludePattern, IncludePattern, exclude_path, adjust_patterns, to_localtime, timestamp, \ - get_cache_dir, get_keys_dir, prune_within, prune_split, \ + get_cache_dir, get_keys_dir, prune_within, prune_split, unhexlify, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ is_cachedir, bigint_to_int, ChunkerParams, CompressionSpec, have_cython, is_slow_msgpack, yes, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR @@ -514,6 +514,28 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_delete_obj(self, args): + """delete the objects with the given IDs from the repo""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + modified = False + for hex_id in args.ids: + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + repository.delete(id) + modified = True + print("object %s deleted." % hex_id) + except repository.ObjectNotFound: + print("object %s not found." % hex_id) + if modified: + repository.commit() + print('Done.') + return EXIT_SUCCESS + helptext = {} helptext['patterns'] = ''' Exclude patterns use a variant of shell pattern syntax, with '*' matching any @@ -1011,6 +1033,20 @@ class Archiver: subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to dump') + + debug_delete_obj_epilog = textwrap.dedent(""" + This command deletes objects from the repository. + """) + subparser = subparsers.add_parser('debug-delete-obj', parents=[common_parser], + description=self.do_debug_delete_obj.__doc__, + epilog=debug_delete_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_delete_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to delete from the repo') return parser def parse_args(self, args=None): From 998670576077ad62645a0fc8e984a0147aaed6d3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 16:51:39 +0100 Subject: [PATCH 4/6] add some tests for the debug commands --- borg/testsuite/archiver.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 61d954e67..f9c438359 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -767,6 +767,31 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_aes_counter_uniqueness_passphrase(self): self.verify_aes_counter_uniqueness('passphrase') + def test_debug_dump_archive_items(self): + self.create_test_files() + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir('output'): + output = self.cmd('debug-dump-archive-items', self.repository_location + '::test') + output_dir = sorted(os.listdir('output')) + assert len(output_dir) > 0 and output_dir[0].startswith('000000_') + assert 'Done.' in output + + def test_debug_delete_obj(self): + self.cmd('init', self.repository_location) + repository = Repository(self.repository_location) + data = b'some data' + h = sha256(data) + key, hexkey = h.digest(), h.hexdigest() + repository.put(key, data) + repository.commit() + output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') + assert "is invalid" in output + output = self.cmd('debug-delete-obj', self.repository_location, hexkey) + assert "deleted" in output + output = self.cmd('debug-delete-obj', self.repository_location, hexkey) + assert "not found" in output + @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase): @@ -875,3 +900,7 @@ class RemoteArchiverTestCase(ArchiverTestCase): @unittest.skip('deadlock issues') def test_fuse_mount_archive(self): pass + + @unittest.skip('only works locally') + def test_debug_delete_obj(self): + pass From a2fc479da38a4e0e49b2cd31ad9ca400cc265188 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 17:31:05 +0100 Subject: [PATCH 5/6] debug-put-obj command --- borg/archiver.py | 28 ++++++++++++++++++++++++++++ borg/testsuite/archiver.py | 17 ++++++++--------- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 9ad291aa1..fe46a20f5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -3,6 +3,7 @@ from .support import argparse # see support/__init__.py docstring from binascii import hexlify from datetime import datetime +from hashlib import sha256 from operator import attrgetter import functools import inspect @@ -514,6 +515,19 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_put_obj(self, args): + """put file(s) contents into the repository""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + for path in args.paths: + with open(path, "rb") as f: + data = f.read() + h = sha256(data) # XXX hardcoded + repository.put(h.digest(), data) + print("object %s put." % h.hexdigest()) + repository.commit() + return EXIT_SUCCESS + def do_debug_delete_obj(self, args): """delete the objects with the given IDs from the repo""" repository = self.open_repository(args.repository) @@ -1034,6 +1048,20 @@ class Archiver: type=location_validator(archive=True), help='archive to dump') + debug_put_obj_epilog = textwrap.dedent(""" + This command puts objects into the repository. + """) + subparser = subparsers.add_parser('debug-put-obj', parents=[common_parser], + description=self.do_debug_put_obj.__doc__, + epilog=debug_put_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_put_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, + help='file(s) to read and create object(s) from') + debug_delete_obj_epilog = textwrap.dedent(""" This command deletes objects from the repository. """) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index f9c438359..717c81aa6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -777,20 +777,19 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert len(output_dir) > 0 and output_dir[0].startswith('000000_') assert 'Done.' in output - def test_debug_delete_obj(self): + def test_debug_put_delete_obj(self): self.cmd('init', self.repository_location) - repository = Repository(self.repository_location) data = b'some data' - h = sha256(data) - key, hexkey = h.digest(), h.hexdigest() - repository.put(key, data) - repository.commit() - output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') - assert "is invalid" in output + hexkey = sha256(data).hexdigest() + self.create_regular_file('file', contents=data) + output = self.cmd('debug-put-obj', self.repository_location, 'input/file') + assert hexkey in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "deleted" in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "not found" in output + output = self.cmd('debug-delete-obj', self.repository_location, 'invalid') + assert "is invalid" in output @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') @@ -902,5 +901,5 @@ class RemoteArchiverTestCase(ArchiverTestCase): pass @unittest.skip('only works locally') - def test_debug_delete_obj(self): + def test_debug_put_delete_obj(self): pass From 37c8aa2d428b2a7170fc894512ffed124318c2cf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 6 Nov 2015 17:45:30 +0100 Subject: [PATCH 6/6] debug-get-obj command --- borg/archiver.py | 36 ++++++++++++++++++++++++++++++++++++ borg/testsuite/archiver.py | 9 +++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index fe46a20f5..1af8e46e5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -515,6 +515,26 @@ class Archiver: print('Done.') return EXIT_SUCCESS + def do_debug_get_obj(self, args): + """get object contents from the repository and write it into file""" + repository = self.open_repository(args.repository) + manifest, key = Manifest.load(repository) + hex_id = args.id + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + data =repository.get(id) + except repository.ObjectNotFound: + print("object %s not found." % hex_id) + else: + with open(args.path, "wb") as f: + f.write(data) + print("object %s fetched." % hex_id) + return EXIT_SUCCESS + def do_debug_put_obj(self, args): """put file(s) contents into the repository""" repository = self.open_repository(args.repository) @@ -1048,6 +1068,22 @@ class Archiver: type=location_validator(archive=True), help='archive to dump') + debug_get_obj_epilog = textwrap.dedent(""" + This command gets an object from the repository. + """) + subparser = subparsers.add_parser('debug-get-obj', parents=[common_parser], + description=self.do_debug_get_obj.__doc__, + epilog=debug_get_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter) + subparser.set_defaults(func=self.do_debug_get_obj) + subparser.add_argument('repository', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('id', metavar='ID', type=str, + help='hex object ID to get from the repo') + subparser.add_argument('path', metavar='PATH', type=str, + help='file to write object data into') + debug_put_obj_epilog = textwrap.dedent(""" This command puts objects into the repository. """) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 717c81aa6..8e7cbf6e8 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -777,13 +777,18 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert len(output_dir) > 0 and output_dir[0].startswith('000000_') assert 'Done.' in output - def test_debug_put_delete_obj(self): + def test_debug_put_get_delete_obj(self): self.cmd('init', self.repository_location) data = b'some data' hexkey = sha256(data).hexdigest() self.create_regular_file('file', contents=data) output = self.cmd('debug-put-obj', self.repository_location, 'input/file') assert hexkey in output + output = self.cmd('debug-get-obj', self.repository_location, hexkey, 'output/file') + assert hexkey in output + with open('output/file', 'rb') as f: + data_read = f.read() + assert data == data_read output = self.cmd('debug-delete-obj', self.repository_location, hexkey) assert "deleted" in output output = self.cmd('debug-delete-obj', self.repository_location, hexkey) @@ -901,5 +906,5 @@ class RemoteArchiverTestCase(ArchiverTestCase): pass @unittest.skip('only works locally') - def test_debug_put_delete_obj(self): + def test_debug_put_get_delete_obj(self): pass