From 07d0a61e4601343025548bf8a6d3193fc8e4e3fa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 1 Jul 2016 04:27:06 +0200 Subject: [PATCH] forced archive deletion, fixes #1139 --- borg/archive.py | 68 +++++++++++++++++++++++++++++++++++++----------- borg/archiver.py | 10 +++++-- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 1ce93ab5d..fad821a0b 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -24,6 +24,8 @@ from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex +from .repository import Repository + import msgpack ITEMS_BUFFER = 1024 * 1024 @@ -494,23 +496,59 @@ def rename(self, name): self.cache.chunk_decref(self.id, self.stats) del self.manifest.archives[self.name] - def delete(self, stats, progress=False): - unpacker = msgpack.Unpacker(use_list=False) - items_ids = self.metadata[b'items'] - pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) - for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + def delete(self, stats, progress=False, forced=False): + class ChunksIndexError(Error): + """Chunk ID {} missing from chunks index, corrupted chunks index - aborting transaction.""" + + def chunk_decref(id, stats): + nonlocal error + try: + self.cache.chunk_decref(id, stats) + except KeyError: + cid = hexlify(id).decode('ascii') + raise ChunksIndexError(cid) + except Repository.ObjectNotFound as e: + # object not in repo - strange, but we wanted to delete it anyway. + if not forced: + raise + error = True + + error = False + try: + unpacker = msgpack.Unpacker(use_list=False) + items_ids = self.metadata[b'items'] + pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", same_line=True) + for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): + if progress: + pi.show(i) + unpacker.feed(self.key.decrypt(items_id, data)) + chunk_decref(items_id, stats) + try: + for item in unpacker: + if b'chunks' in item: + for chunk_id, size, csize in item[b'chunks']: + chunk_decref(chunk_id, stats) + except (TypeError, ValueError): + # if items metadata spans multiple chunks and one chunk got dropped somehow, + # it could be that unpacker yields bad types + if not forced: + raise + error = True if progress: - pi.show(i) - unpacker.feed(self.key.decrypt(items_id, data)) - self.cache.chunk_decref(items_id, stats) - for item in unpacker: - if b'chunks' in item: - for chunk_id, size, csize in item[b'chunks']: - self.cache.chunk_decref(chunk_id, stats) - if progress: - pi.finish() - self.cache.chunk_decref(self.id, stats) + pi.finish() + except (msgpack.UnpackException, Repository.ObjectNotFound): + # items metadata corrupted + if not forced: + raise + error = True + # in forced delete mode, we try hard to delete at least the manifest entry, + # if possible also the archive superblock, even if processing the items raises + # some harmless exception. + chunk_decref(self.id, stats) del self.manifest.archives[self.name] + if error: + logger.warning('forced deletion succeeded, but the deleted archive was corrupted.') + logger.warning('borg check --repair is required to free all space.') def stat_attrs(self, st, path): item = { diff --git a/borg/archiver.py b/borg/archiver.py index 49e0bdbaa..e5f1af91d 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -426,7 +426,7 @@ def do_delete(self, args, repository): with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache) stats = Statistics() - archive.delete(stats, progress=args.progress) + archive.delete(stats, progress=args.progress, forced=args.forced) manifest.write() repository.commit(save_space=args.save_space) cache.commit() @@ -635,7 +635,7 @@ def do_prune(self, args, repository, manifest, key): else: if args.output_list: logger.info('Pruning archive: %s' % format_archive(archive)) - Archive(repository, key, manifest, archive.name, cache).delete(stats) + Archive(repository, key, manifest, archive.name, cache).delete(stats, forced=args.forced) if to_delete and not args.dry_run: manifest.write() repository.commit(save_space=args.save_space) @@ -1230,6 +1230,9 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('-c', '--cache-only', dest='cache_only', action='store_true', default=False, help='delete only the local cache for the given repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force deletion of corrupted archives') subparser.add_argument('--save-space', dest='save_space', action='store_true', default=False, help='work slower, but using less space') @@ -1342,6 +1345,9 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not change repository') + subparser.add_argument('--force', dest='forced', + action='store_true', default=False, + help='force pruning of corrupted archives') subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the deleted archive')