From 8a1ebe0112aad0f1e087aadbd33e8579628cb96d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Wed, 19 Mar 2014 22:32:07 +0100 Subject: [PATCH] Added '--stats' option to attic prune and attic delete --- CHANGES | 1 + attic/archive.py | 14 +++++++------- attic/archiver.py | 25 +++++++++++++++++++------ attic/cache.py | 4 +++- attic/helpers.py | 16 +++++++--------- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/CHANGES b/CHANGES index cfb8e00ae..394b5abdf 100644 --- a/CHANGES +++ b/CHANGES @@ -10,6 +10,7 @@ Version 0.12 - Include "all archives" size information in "--stats" output. (#54) - Switch to SI units (Power of 1000 instead 1024) when printing file sizes +- Added "--stats" option to 'attic delete' and 'attic prune' Version 0.11 ------------ diff --git a/attic/archive.py b/attic/archive.py index 59acfdfed..868d79179 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -171,7 +171,7 @@ class Archive: def write_checkpoint(self): self.save(self.checkpoint_name) del self.manifest.archives[self.checkpoint_name] - self.cache.chunk_decref(self.id) + self.cache.chunk_decref(self.id, self.stats) def save(self, name=None): name = name or self.name @@ -316,17 +316,17 @@ class Archive: elif not symlink: os.utime(path, (item[b'mtime'] / 10**9, item[b'mtime'] / 10**9)) - def delete(self): + def delete(self, stats): unpacker = msgpack.Unpacker(use_list=False) - for id_, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): - unpacker.feed(self.key.decrypt(id_, data)) - self.cache.chunk_decref(id_) + for items_id, data in zip(self.metadata[b'items'], self.repository.get_many(self.metadata[b'items'])): + unpacker.feed(self.key.decrypt(items_id, data)) + self.cache.chunk_decref(items_id, stats) for item in unpacker: if b'chunks' in item: for chunk_id, size, csize in item[b'chunks']: - self.cache.chunk_decref(chunk_id) + self.cache.chunk_decref(chunk_id, stats) - self.cache.chunk_decref(self.id) + self.cache.chunk_decref(self.id, stats) del self.manifest.archives[self.name] def stat_attrs(self, st, path): diff --git a/attic/archiver.py b/attic/archiver.py index 20e4625cc..b33abc4a7 100644 --- a/attic/archiver.py +++ b/attic/archiver.py @@ -16,7 +16,7 @@ from attic.key import key_creator from attic.helpers import Error, location_validator, format_time, \ format_file_mode, ExcludePattern, exclude_path, adjust_patterns, to_localtime, \ get_cache_dir, get_keys_dir, format_timedelta, prune_within, prune_split, \ - Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules + Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics from attic.remote import RepositoryServer, RemoteRepository @@ -136,7 +136,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print('Start time: %s' % t0.strftime('%c')) print('End time: %s' % t.strftime('%c')) print('Duration: %s' % format_timedelta(diff)) - archive.stats.print_(cache) + print('Number of files: %d' % archive.stats.nfiles) + archive.stats.print_('This archive:', cache) print('-' * 78) return self.exit_code @@ -219,10 +220,13 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") manifest, key = Manifest.load(repository) cache = Cache(repository, key, manifest) archive = Archive(repository, key, manifest, args.archive.archive, cache=cache) - archive.delete() + stats = Statistics() + archive.delete(stats) manifest.write() repository.commit() cache.commit() + if args.stats: + stats.print_('Deleted data:', cache) return self.exit_code def do_mount(self, args): @@ -300,7 +304,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print('Username:', archive.metadata[b'username']) print('Time: %s' % to_localtime(archive.ts).strftime('%c')) print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) - stats.print_(cache) + print('Number of files: %d' % archive.stats.nfiles) + stats.print_('This archive:', cache) return self.exit_code def do_prune(self, args): @@ -333,7 +338,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") keep.sort(key=attrgetter('ts'), reverse=True) to_delete = [a for a in archives if a not in keep] - + stats = Statistics() for archive in keep: self.print_verbose('Keeping archive: %s' % format_archive(archive)) for archive in to_delete: @@ -341,11 +346,13 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") self.print_verbose('Would prune: %s' % format_archive(archive)) else: self.print_verbose('Pruning archive: %s' % format_archive(archive)) - archive.delete() + archive.delete(stats) if to_delete and not args.dry_run: manifest.write() repository.commit() cache.commit() + if args.stats: + stats.print_('Deleted data:', cache) return self.exit_code helptext = {} @@ -530,6 +537,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser = subparsers.add_parser('delete', parents=[common_parser], description=self.do_delete.__doc__) subparser.set_defaults(func=self.do_delete) + subparser.add_argument('-s', '--stats', dest='stats', + action='store_true', default=False, + help='print statistics for the deleted archive') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to delete') @@ -586,6 +596,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not change repository') + subparser.add_argument('-s', '--stats', dest='stats', + action='store_true', default=False, + help='print statistics for the deleted archive') subparser.add_argument('--keep-within', dest='within', type=str, metavar='WITHIN', help='keep all archives within this time interval') subparser.add_argument('-H', '--keep-hourly', dest='hourly', type=int, default=0, diff --git a/attic/cache.py b/attic/cache.py index d029394f8..1bf311b0b 100644 --- a/attic/cache.py +++ b/attic/cache.py @@ -193,15 +193,17 @@ class Cache(object): stats.update(size, csize, False) return id, size, csize - def chunk_decref(self, id): + def chunk_decref(self, id, stats): if not self.txn_active: self.begin_txn() count, size, csize = self.chunks[id] if count == 1: del self.chunks[id] self.repository.delete(id, wait=False) + stats.update(-size, -csize, True) else: self.chunks[id] = (count - 1, size, csize) + stats.update(-size, -csize, False) def file_known_and_unchanged(self, path_hash, st): if self.files is None: diff --git a/attic/helpers.py b/attic/helpers.py index e599636ec..e787e8f51 100644 --- a/attic/helpers.py +++ b/attic/helpers.py @@ -147,16 +147,14 @@ class Statistics: if unique: self.usize += csize - def print_(self, cache): + def print_(self, label, cache): total_size, total_csize, unique_size, unique_csize = cache.chunks.summarize() - print('Number of files: %d' % self.nfiles) print() print(' Original size Compressed size Deduplicated size') - print('This archive: %20s %20s %20s' % (format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize))) + print('%-15s %20s %20s %20s' % (label, format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize))) print('All archives: %20s %20s %20s' % (format_file_size(total_size), format_file_size(total_csize), format_file_size(unique_csize))) - def get_keys_dir(): """Determine where to repository keys and cache""" return os.environ.get('ATTIC_KEYS_DIR', @@ -296,16 +294,16 @@ def format_file_mode(mod): def format_file_size(v): """Format file size into a human friendly format """ - if v > 10**12: + if abs(v) > 10**12: return '%.2f TB' % (v / 10**12) - elif v > 10**9: + elif abs(v) > 10**9: return '%.2f GB' % (v / 10**9) - elif v > 10**6: + elif abs(v) > 10**6: return '%.2f MB' % (v / 10**6) - elif v > 10**3: + elif abs(v) > 10**3: return '%.2f kB' % (v / 10**3) else: - return '%d B ' % v + return '%d B' % v def format_archive(archive):