From cc26bdf8102641b7ac8d4fee376a9b06c5994125 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 11:41:06 +0100 Subject: [PATCH 1/9] info: add --json option --- src/borg/archiver.py | 58 ++++++++++++++++++++++++++-------- src/borg/cache.py | 18 +++++++---- src/borg/testsuite/archiver.py | 10 ++++++ 3 files changed, 65 insertions(+), 21 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 041749a19..f45ef2ab0 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -65,6 +65,10 @@ from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader STATS_HEADER = " Original size Compressed size Deduplicated size" +def print_as_json(obj): + print(json.dumps(obj, sort_keys=True, indent=4)) + + def argument(args, str_or_bool): """If bool is passed, return it. If str is passed, retrieve named attribute from args.""" if isinstance(str_or_bool, str): @@ -960,7 +964,7 @@ class Archiver: if any((args.location.archive, args.first, args.last, args.prefix)): return self._info_archives(args, repository, manifest, key, cache) else: - return self._info_repository(repository, key, cache) + return self._info_repository(args, repository, key, cache) def _info_archives(self, args, repository, manifest, key, cache): def format_cmdline(cmdline): @@ -998,20 +1002,44 @@ class Archiver: print() return self.exit_code - def _info_repository(self, repository, key, cache): - print('Repository ID: %s' % bin_to_hex(repository.id)) - if key.NAME == 'plaintext': - encrypted = 'No' + def _info_repository(self, args, repository, key, cache): + if args.json: + encryption = { + 'mode': key.NAME, + } + if key.NAME.startswith('key file'): + encryption['keyfile'] = key.find_key() else: - encrypted = 'Yes (%s)' % key.NAME - print('Encrypted: %s' % encrypted) - if key.NAME.startswith('key file'): - print('Key file: %s' % key.find_key()) - print('Cache: %s' % cache.path) - print('Security dir: %s' % cache.security_manager.dir) - print(DASHES) - print(STATS_HEADER) - print(str(cache)) + encryption = 'Encrypted: ' + if key.NAME == 'plaintext': + encryption += 'No' + else: + encryption += 'Yes (%s)' % key.NAME + if key.NAME.startswith('key file'): + encryption += '\nKey file: %s' % key.find_key() + + info = { + 'id': bin_to_hex(repository.id), + 'location': repository._location.canonical_path(), + 'cache': cache.path, + 'security_dir': cache.security_manager.dir, + 'encryption': encryption, + } + + if args.json: + info['cache-stats'] = cache.stats() + print_as_json(info) + else: + print(textwrap.dedent(""" + Repository ID: {id} + Location: {location} + {encryption} + Cache: {cache} + Security dir: {security_dir} + """).strip().format_map(info)) + print(DASHES) + print(STATS_HEADER) + print(str(cache)) return self.exit_code @with_repository(exclusive=True) @@ -2542,6 +2570,8 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='archive or repository to display information about') + subparser.add_argument('--json', action='store_true', + help='format output as JSON') self.add_archives_filters_args(subparser) break_lock_epilog = process_epilog(""" diff --git a/src/borg/cache.py b/src/borg/cache.py index 21efcbc9e..f1a8ef85e 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -219,18 +219,22 @@ All archives: {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s} Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" return fmt.format(self.format_tuple()) - def format_tuple(self): + Summary = namedtuple('Summary', ['total_size', 'total_csize', 'unique_size', 'unique_csize', 'total_unique_chunks', + 'total_chunks']) + + def stats(self): # XXX: this should really be moved down to `hashindex.pyx` - Summary = namedtuple('Summary', ['total_size', 'total_csize', 'unique_size', 'unique_csize', 'total_unique_chunks', 'total_chunks']) - stats = Summary(*self.chunks.summarize())._asdict() + stats = self.Summary(*self.chunks.summarize())._asdict() + return stats + + def format_tuple(self): + stats = self.stats() for field in ['total_size', 'total_csize', 'unique_csize']: stats[field] = format_file_size(stats[field]) - return Summary(**stats) + return self.Summary(**stats) def chunks_stored_size(self): - Summary = namedtuple('Summary', ['total_size', 'total_csize', 'unique_size', 'unique_csize', 'total_unique_chunks', 'total_chunks']) - stats = Summary(*self.chunks.summarize()) - return stats.unique_csize + return self.stats()['unique_csize'] def create(self): """Create a new empty cache at `self.path` diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index cce0c92ff..5e6dad87d 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1112,6 +1112,16 @@ class ArchiverTestCase(ArchiverTestCaseBase): info_archive = self.cmd('info', '--first', '1', self.repository_location) assert 'Archive name: test\n' in info_archive + def test_info_json(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + info_repo = json.loads(self.cmd('info', '--json', self.repository_location)) + assert len(info_repo['id']) == 64 + assert info_repo['encryption']['mode'] == 'repokey' + assert 'keyfile' not in info_repo['encryption'] + assert 'cache-stats' in info_repo + def test_comment(self): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', '--encryption=repokey', self.repository_location) From 7cbade2f8c117fcab59b9dbbb0152e28ea6abd11 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 11:54:57 +0100 Subject: [PATCH 2/9] create: add --json option --- src/borg/archive.py | 21 +++++++++++++++++++++ src/borg/archiver.py | 25 +++++++++++++++++-------- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 07d62e168..2073080a6 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -68,6 +68,14 @@ class Statistics: return "<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>".format( cls=type(self).__name__, hash=id(self), self=self) + def as_dict(self): + return { + 'original_size': self.osize, + 'compressed_size': self.csize, + 'deduplicated_size': self.usize, + 'nfiles': self.nfiles, + } + @property def osize_fmt(self): return format_file_size(self.osize) @@ -343,6 +351,19 @@ class Archive: def duration_from_meta(self): return format_timedelta(self.ts_end - self.ts) + def info(self): + return { + 'name': self.name, + 'id': self.fpr, + 'start': format_time(to_localtime(self.start.replace(tzinfo=timezone.utc))), + 'end': format_time(to_localtime(self.end.replace(tzinfo=timezone.utc))), + 'duration': (self.end - self.start).total_seconds(), + 'nfiles': self.stats.nfiles, + 'limits': { + 'max_archive_size': self.cache.chunks[self.id].csize / MAX_DATA_SIZE, + }, + } + def __str__(self): return '''\ Archive name: {0.name} diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f45ef2ab0..76ea89927 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -369,13 +369,20 @@ class Archiver: if args.progress: archive.stats.show_progress(final=True) if args.stats: - log_multi(DASHES, - str(archive), - DASHES, - STATS_HEADER, - str(archive.stats), - str(cache), - DASHES, logger=logging.getLogger('borg.output.stats')) + if args.json: + print_as_json({ + 'cache_stats': cache.stats(), + 'stats': archive.stats.as_dict(), + 'archive': archive.info(), + }) + else: + log_multi(DASHES, + str(archive), + DASHES, + STATS_HEADER, + str(archive.stats), + str(cache), + DASHES, logger=logging.getLogger('borg.output.stats')) self.output_filter = args.output_filter self.output_list = args.output_list @@ -1027,7 +1034,7 @@ class Archiver: } if args.json: - info['cache-stats'] = cache.stats() + info['cache_stats'] = cache.stats() print_as_json(info) else: print(textwrap.dedent(""" @@ -2174,6 +2181,8 @@ class Archiver: help='output verbose list of items (files, dirs, ...)') subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', help='only display items with the given status characters') + subparser.add_argument('--json', action='store_true', + help='output stats as JSON') exclude_group = subparser.add_argument_group('Exclusion options') exclude_group.add_argument('-e', '--exclude', dest='patterns', From 2ab5d0f2139884ea873f5e7324213ed10c53a908 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 12:09:09 +0100 Subject: [PATCH 3/9] use custom JSON encoder for repr'ing Borg objects consistently --- src/borg/archiver.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 76ea89927..0f9594569 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -65,8 +65,25 @@ from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader STATS_HEADER = " Original size Compressed size Deduplicated size" +class BorgJsonEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, Repository) or isinstance(o, RemoteRepository): + return { + 'id': bin_to_hex(o.id), + 'location': o._location.canonical_path(), + } + if isinstance(o, Archive): + return o.info() + if isinstance(o, Cache): + return { + 'path': o.path, + 'stats': o.stats(), + } + return super().default(o) + + def print_as_json(obj): - print(json.dumps(obj, sort_keys=True, indent=4)) + print(json.dumps(obj, sort_keys=True, indent=4, cls=BorgJsonEncoder)) def argument(args, str_or_bool): @@ -371,9 +388,10 @@ class Archiver: if args.stats: if args.json: print_as_json({ - 'cache_stats': cache.stats(), + 'repository': repository, + 'cache': cache, 'stats': archive.stats.as_dict(), - 'archive': archive.info(), + 'archive': archive, }) else: log_multi(DASHES, @@ -1026,9 +1044,8 @@ class Archiver: encryption += '\nKey file: %s' % key.find_key() info = { - 'id': bin_to_hex(repository.id), - 'location': repository._location.canonical_path(), - 'cache': cache.path, + 'repository': repository, + 'cache': cache, 'security_dir': cache.security_manager.dir, 'encryption': encryption, } @@ -1041,9 +1058,12 @@ class Archiver: Repository ID: {id} Location: {location} {encryption} - Cache: {cache} + Cache: {cache.path} Security dir: {security_dir} - """).strip().format_map(info)) + """).strip().format( + id=bin_to_hex(repository.id), + location=repository._location.canonical_path(), + **info)) print(DASHES) print(STATS_HEADER) print(str(cache)) From 6180f5055c3f4c1b60bbf1c85f59afaa8d8f5a27 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 12:28:01 +0100 Subject: [PATCH 4/9] info: --json for archives --- src/borg/archive.py | 30 ++++++++++++++++++++++------ src/borg/archiver.py | 47 +++++++++++++++++++++++++++----------------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 2073080a6..7b87c0bdf 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -290,7 +290,8 @@ class Archive: self.end = end self.consider_part_files = consider_part_files self.pipeline = DownloadPipeline(self.repository, self.key) - if create: + self.create = create + if self.create: self.file_compression_logger = create_logger('borg.debug.file-compression') self.items_buffer = CacheChunkBuffer(self.cache, self.key, self.stats) self.chunker = Chunker(self.key.chunk_seed, *chunker_params) @@ -352,17 +353,34 @@ class Archive: return format_timedelta(self.ts_end - self.ts) def info(self): - return { + if self.create: + stats = self.stats + start = self.start.replace(tzinfo=timezone.utc) + end = self.end.replace(tzinfo=timezone.utc) + else: + stats = self.calc_stats(self.cache) + start = self.ts + end = self.ts_end + info = { 'name': self.name, 'id': self.fpr, - 'start': format_time(to_localtime(self.start.replace(tzinfo=timezone.utc))), - 'end': format_time(to_localtime(self.end.replace(tzinfo=timezone.utc))), - 'duration': (self.end - self.start).total_seconds(), - 'nfiles': self.stats.nfiles, + 'start': format_time(to_localtime(start)), + 'end': format_time(to_localtime(end)), + 'duration': (end - start).total_seconds(), + 'stats': stats.as_dict(), 'limits': { 'max_archive_size': self.cache.chunks[self.id].csize / MAX_DATA_SIZE, }, } + if self.create: + info['command_line'] = sys.argv + else: + info.update({ + 'command_line': self.metadata.cmdline, + 'hostname': self.metadata.hostname, + 'username': self.metadata.username, + }) + return info def __str__(self): return '''\ diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 0f9594569..214b57469 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -390,7 +390,6 @@ class Archiver: print_as_json({ 'repository': repository, 'cache': cache, - 'stats': archive.stats.as_dict(), 'archive': archive, }) else: @@ -1002,29 +1001,41 @@ class Archiver: if not archive_names: return self.exit_code + output_data = [] + for i, archive_name in enumerate(archive_names, 1): archive = Archive(repository, key, manifest, archive_name, cache=cache, consider_part_files=args.consider_part_files) - stats = archive.calc_stats(cache) - print('Archive name: %s' % archive.name) - print('Archive fingerprint: %s' % archive.fpr) - print('Comment: %s' % archive.metadata.get('comment', '')) - print('Hostname: %s' % archive.metadata.hostname) - print('Username: %s' % archive.metadata.username) - print('Time (start): %s' % format_time(to_localtime(archive.ts))) - print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) - print('Duration: %s' % archive.duration_from_meta) - print('Number of files: %d' % stats.nfiles) - print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) - print('Utilization of max. archive size: %d%%' % (100 * cache.chunks[archive.id].csize / MAX_DATA_SIZE)) - print(DASHES) - print(STATS_HEADER) - print(str(stats)) - print(str(cache)) + if args.json: + output_data.append(archive.info()) + else: + stats = archive.calc_stats(cache) + print('Archive name: %s' % archive.name) + print('Archive fingerprint: %s' % archive.fpr) + print('Comment: %s' % archive.metadata.get('comment', '')) + print('Hostname: %s' % archive.metadata.hostname) + print('Username: %s' % archive.metadata.username) + print('Time (start): %s' % format_time(to_localtime(archive.ts))) + print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) + print('Duration: %s' % archive.duration_from_meta) + print('Number of files: %d' % stats.nfiles) + print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) + print('Utilization of max. archive size: %d%%' % (100 * cache.chunks[archive.id].csize / MAX_DATA_SIZE)) + print(DASHES) + print(STATS_HEADER) + print(str(stats)) + print(str(cache)) if self.exit_code: break - if len(archive_names) - i: + if not args.json and len(archive_names) - i: print() + + if args.json: + print_as_json({ + 'repository': repository, + 'cache': cache, + 'archives': output_data, + }) return self.exit_code def _info_repository(self, args, repository, key, cache): From 25781f53d4c4d76a4070003f96eb6d9906337578 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 12:32:26 +0100 Subject: [PATCH 5/9] list: --json for archive listing --- src/borg/archiver.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 214b57469..e380917b4 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -977,8 +977,19 @@ class Archiver: format = "{archive:<36} {time} [{id}]{NL}" formatter = ArchiveFormatter(format) + output_data = [] + for archive_info in manifest.archives.list_considering(args): - write(safe_encode(formatter.format_item(archive_info))) + if args.json: + output_data.append(formatter.get_item_data(archive_info)) + else: + write(safe_encode(formatter.format_item(archive_info))) + + if args.json: + print_as_json({ + 'repository': manifest.repository, + 'archives': output_data, + }) return self.exit_code @@ -2492,6 +2503,8 @@ class Archiver: subparser.add_argument('--format', '--list-format', dest='format', type=str, help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") + subparser.add_argument('--json', action='store_true', + help='format output as JSON') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository/archive to list contents of') From 1f8c0929bf5013d40c27e9cba84de9c8915c022b Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 12:51:57 +0100 Subject: [PATCH 6/9] list: --json for archive contents listing --- src/borg/archiver.py | 8 ++++++-- src/borg/helpers.py | 42 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index e380917b4..f0e6d7e08 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -962,10 +962,12 @@ class Archiver: format = "{path}{NL}" else: format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" - formatter = ItemFormatter(archive, format) + formatter = ItemFormatter(archive, format, json=args.json) + write(safe_encode(formatter.begin())) for item in archive.iter_items(lambda item: matcher.match(item.path)): write(safe_encode(formatter.format_item(item))) + write(safe_encode(formatter.end())) return self.exit_code def _list_repository(self, args, manifest, write): @@ -2504,7 +2506,9 @@ class Archiver: help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") subparser.add_argument('--json', action='store_true', - help='format output as JSON') + help='format output as JSON. The form of --format is ignored, but keys used in it ' + 'are added to the JSON output. Some keys are always present. Note: JSON can only ' + 'represent text. A "bpath" key is therefore not available.') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository/archive to list contents of') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 89b557e57..9964cb448 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -5,6 +5,7 @@ import grp import hashlib import logging import io +import json import os import os.path import platform @@ -1620,8 +1621,9 @@ class ItemFormatter(BaseFormatter): assert not keys, str(keys) return "\n".join(help) - def __init__(self, archive, format): + def __init__(self, archive, format, *, json=False): self.archive = archive + self.json = json static_keys = { 'archivename': archive.name, 'archiveid': archive.fpr, @@ -1646,7 +1648,34 @@ class ItemFormatter(BaseFormatter): for hash_function in hashlib.algorithms_guaranteed: self.add_key(hash_function, partial(self.hash_item, hash_function)) self.used_call_keys = set(self.call_keys) & self.format_keys - self.item_data = static_keys + if self.json: + self.item_data = {} + self.format_item = self.format_item_json + self.first = True + else: + self.item_data = static_keys + + def begin(self): + from borg.archiver import BorgJsonEncoder + if not self.json: + return '' + return textwrap.dedent(""" + {{ + "repository": {repository}, + "files": [ + """).strip().format(repository=BorgJsonEncoder().encode(self.archive.repository)) + + def end(self): + if not self.json: + return '' + return "]}" + + def format_item_json(self, item): + if self.first: + self.first = False + return json.dumps(self.get_item_data(item)) + else: + return ',' + json.dumps(self.get_item_data(item)) def add_key(self, key, callable_with_item): self.call_keys[key] = callable_with_item @@ -1673,12 +1702,15 @@ class ItemFormatter(BaseFormatter): item_data['uid'] = item.uid item_data['gid'] = item.gid item_data['path'] = remove_surrogates(item.path) - item_data['bpath'] = item.path + if self.json: + item_data['healthy'] = 'chunks_healthy' not in item + else: + item_data['bpath'] = item.path + item_data['extra'] = extra + item_data['health'] = 'broken' if 'chunks_healthy' in item else 'healthy' item_data['source'] = source item_data['linktarget'] = source - item_data['extra'] = extra item_data['flags'] = item.get('bsdflags') - item_data['health'] = 'broken' if 'chunks_healthy' in item else 'healthy' for key in self.used_call_keys: item_data[key] = self.call_keys[key](item) return item_data From 56563a4392600f096e5966574b44f9132cb43823 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 14:27:32 +0100 Subject: [PATCH 7/9] move JSON generation and utilities to helpers --- src/borg/archiver.py | 70 +++++++++------------------------- src/borg/helpers.py | 64 +++++++++++++++++++++++++++---- src/borg/testsuite/archiver.py | 65 ++++++++++++++++++++++++++++++- 3 files changed, 138 insertions(+), 61 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f0e6d7e08..39504dc02 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -52,6 +52,7 @@ from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm from .helpers import ErrorIgnoringTextIOWrapper from .helpers import ProgressIndicatorPercent +from .helpers import BorgJsonEncoder, basic_json_data, json_print from .item import Item from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey from .keymanager import KeyManager @@ -65,27 +66,6 @@ from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader STATS_HEADER = " Original size Compressed size Deduplicated size" -class BorgJsonEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, Repository) or isinstance(o, RemoteRepository): - return { - 'id': bin_to_hex(o.id), - 'location': o._location.canonical_path(), - } - if isinstance(o, Archive): - return o.info() - if isinstance(o, Cache): - return { - 'path': o.path, - 'stats': o.stats(), - } - return super().default(o) - - -def print_as_json(obj): - print(json.dumps(obj, sort_keys=True, indent=4, cls=BorgJsonEncoder)) - - def argument(args, str_or_bool): """If bool is passed, return it. If str is passed, retrieve named attribute from args.""" if isinstance(str_or_bool, str): @@ -385,13 +365,12 @@ class Archiver: archive.save(comment=args.comment, timestamp=args.timestamp) if args.progress: archive.stats.show_progress(final=True) + args.stats |= args.json if args.stats: if args.json: - print_as_json({ - 'repository': repository, - 'cache': cache, + json_print(basic_json_data(manifest, cache=cache, extra={ 'archive': archive, - }) + })) else: log_multi(DASHES, str(archive), @@ -988,10 +967,9 @@ class Archiver: write(safe_encode(formatter.format_item(archive_info))) if args.json: - print_as_json({ - 'repository': manifest.repository, - 'archives': output_data, - }) + json_print(basic_json_data(manifest, extra={ + 'archives': output_data + })) return self.exit_code @@ -1001,7 +979,7 @@ class Archiver: if any((args.location.archive, args.first, args.last, args.prefix)): return self._info_archives(args, repository, manifest, key, cache) else: - return self._info_repository(args, repository, key, cache) + return self._info_repository(args, repository, manifest, key, cache) def _info_archives(self, args, repository, manifest, key, cache): def format_cmdline(cmdline): @@ -1044,20 +1022,18 @@ class Archiver: print() if args.json: - print_as_json({ - 'repository': repository, - 'cache': cache, + json_print(basic_json_data(manifest, cache=cache, extra={ 'archives': output_data, - }) + })) return self.exit_code - def _info_repository(self, args, repository, key, cache): + def _info_repository(self, args, repository, manifest, key, cache): + info = basic_json_data(manifest, cache=cache, extra={ + 'security_dir': cache.security_manager.dir, + }) + if args.json: - encryption = { - 'mode': key.NAME, - } - if key.NAME.startswith('key file'): - encryption['keyfile'] = key.find_key() + json_print(info) else: encryption = 'Encrypted: ' if key.NAME == 'plaintext': @@ -1066,18 +1042,8 @@ class Archiver: encryption += 'Yes (%s)' % key.NAME if key.NAME.startswith('key file'): encryption += '\nKey file: %s' % key.find_key() + info['encryption'] = encryption - info = { - 'repository': repository, - 'cache': cache, - 'security_dir': cache.security_manager.dir, - 'encryption': encryption, - } - - if args.json: - info['cache_stats'] = cache.stats() - print_as_json(info) - else: print(textwrap.dedent(""" Repository ID: {id} Location: {location} @@ -2226,7 +2192,7 @@ class Archiver: subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS', help='only display items with the given status characters') subparser.add_argument('--json', action='store_true', - help='output stats as JSON') + help='output stats as JSON (implies --stats)') exclude_group = subparser.add_argument_group('Exclusion options') exclude_group.add_argument('-e', '--exclude', dest='patterns', diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 9964cb448..39bbf01a6 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -207,6 +207,10 @@ class Manifest: def id_str(self): return bin_to_hex(self.id) + @property + def last_timestamp(self): + return datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + @classmethod def load(cls, repository, key=None, force_tam_not_required=False): from .item import ManifestItem @@ -251,7 +255,7 @@ class Manifest: if self.timestamp is None: self.timestamp = datetime.utcnow().isoformat() else: - prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + prev_ts = self.last_timestamp incremented = (prev_ts + timedelta(microseconds=1)).isoformat() self.timestamp = max(incremented, datetime.utcnow().isoformat()) manifest = ManifestItem( @@ -1656,14 +1660,13 @@ class ItemFormatter(BaseFormatter): self.item_data = static_keys def begin(self): - from borg.archiver import BorgJsonEncoder if not self.json: return '' - return textwrap.dedent(""" - {{ - "repository": {repository}, - "files": [ - """).strip().format(repository=BorgJsonEncoder().encode(self.archive.repository)) + begin = json_dump(basic_json_data(self.archive.manifest)) + begin, _, _ = begin.rpartition('\n}') # remove last closing brace, we want to extend the object + begin += ',\n' + begin += ' "files": [\n' + return begin def end(self): if not self.json: @@ -2090,3 +2093,50 @@ def swidth_slice(string, max_width): if reverse: result.reverse() return ''.join(result) + + +class BorgJsonEncoder(json.JSONEncoder): + def default(self, o): + from .repository import Repository + from .remote import RemoteRepository + from .archive import Archive + from .cache import Cache + if isinstance(o, Repository) or isinstance(o, RemoteRepository): + return { + 'id': bin_to_hex(o.id), + 'location': o._location.canonical_path(), + } + if isinstance(o, Archive): + return o.info() + if isinstance(o, Cache): + return { + 'path': o.path, + 'stats': o.stats(), + } + return super().default(o) + + +def basic_json_data(manifest, *, cache=None, extra=None): + key = manifest.key + data = extra or {} + data.update({ + 'repository': BorgJsonEncoder().default(manifest.repository), + 'encryption': { + 'mode': key.NAME, + }, + }) + data['repository']['last_modified'] = format_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) + if key.NAME.startswith('key file'): + data['encryption']['keyfile'] = key.find_key() + if cache: + data['cache'] = cache + return data + + +def json_dump(obj): + """Dump using BorgJSONEncoder.""" + return json.dumps(obj, sort_keys=True, indent=4, cls=BorgJsonEncoder) + + +def json_print(obj): + print(json_dump(obj)) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 5e6dad87d..ca0f41028 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1117,10 +1117,27 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') info_repo = json.loads(self.cmd('info', '--json', self.repository_location)) - assert len(info_repo['id']) == 64 + repository = info_repo['repository'] + assert len(repository['id']) == 64 + assert 'last_modified' in repository assert info_repo['encryption']['mode'] == 'repokey' assert 'keyfile' not in info_repo['encryption'] - assert 'cache-stats' in info_repo + cache = info_repo['cache'] + stats = cache['stats'] + assert all(isinstance(o, int) for o in stats.values()) + assert all(key in stats for key in ('total_chunks', 'total_csize', 'total_size', 'total_unique_chunks', 'unique_csize', 'unique_size')) + + info_archive = json.loads(self.cmd('info', '--json', self.repository_location + '::test')) + assert info_repo['repository'] == info_archive['repository'] + assert info_repo['cache'] == info_archive['cache'] + archives = info_archive['archives'] + assert len(archives) == 1 + archive = archives[0] + assert archive['name'] == 'test' + assert isinstance(archive['command_line'], list) + assert isinstance(archive['duration'], float) + assert len(archive['id']) == 64 + assert 'stats' in archive def test_comment(self): self.create_regular_file('file1', size=1024 * 80) @@ -1273,6 +1290,23 @@ class ArchiverTestCase(ArchiverTestCaseBase): if has_lchflags: self.assert_in("x input/file3", output) + def test_create_json(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', '--encryption=repokey', self.repository_location) + create_info = json.loads(self.cmd('create', '--json', self.repository_location + '::test', 'input')) + # The usual keys + assert 'encryption' in create_info + assert 'repository' in create_info + assert 'cache' in create_info + assert 'last_modified' in create_info['repository'] + + archive = create_info['archive'] + assert archive['name'] == 'test' + assert isinstance(archive['command_line'], list) + assert isinstance(archive['duration'], float) + assert len(archive['id']) == 64 + assert 'stats' in archive + def test_create_topical(self): now = time.time() self.create_regular_file('file1', size=1024 * 80) @@ -1457,6 +1491,33 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert int(dsize) <= int(size) assert int(dcsize) <= int(csize) + def test_list_json(self): + self.create_regular_file('file1', size=1024 * 80) + self.cmd('init', '--encryption=repokey', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + list_repo = json.loads(self.cmd('list', '--json', self.repository_location)) + repository = list_repo['repository'] + assert len(repository['id']) == 64 + assert 'last_modified' in repository + assert list_repo['encryption']['mode'] == 'repokey' + assert 'keyfile' not in list_repo['encryption'] + + list_archive = json.loads(self.cmd('list', '--json', self.repository_location + '::test')) + assert list_repo['repository'] == list_archive['repository'] + files = list_archive['files'] + assert len(files) == 2 + file1 = files[1] + assert file1['path'] == 'input/file1' + assert file1['size'] == 81920 + + list_archive = json.loads(self.cmd('list', '--json', '--format={sha256}', self.repository_location + '::test')) + assert list_repo['repository'] == list_archive['repository'] + files = list_archive['files'] + assert len(files) == 2 + file1 = files[1] + assert file1['path'] == 'input/file1' + assert file1['sha256'] == 'b2915eb69f260d8d3c25249195f2c8f4f716ea82ec760ae929732c0262442b2b' + def _get_sizes(self, compression, compressible, size=10000): if compressible: contents = b'X' * size From 8cdf192511b46d58bf298a2efc084a5ec1e9c12a Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 15:17:17 +0100 Subject: [PATCH 8/9] list: add "name" key for consistency with info cmd --- src/borg/helpers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 39bbf01a6..792e34571 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1558,6 +1558,7 @@ class ArchiveFormatter(BaseFormatter): def get_item_data(self, archive): return { + 'name': remove_surrogates(archive.name), 'barchive': archive.name, 'archive': remove_surrogates(archive.name), 'id': bin_to_hex(archive.id), @@ -1566,7 +1567,7 @@ class ArchiveFormatter(BaseFormatter): @staticmethod def keys_help(): - return " - archive: archive name interpreted as text (might be missing non-text characters, see barchive)\n" \ + return " - archive, name: archive name interpreted as text (might be missing non-text characters, see barchive)\n" \ " - barchive: verbatim archive name, can contain any character except NUL\n" \ " - time: time of creation of the archive\n" \ " - id: internal ID of the archive" From 4f1db82f6d45e481acfaf0d302b7554d1e773238 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Thu, 23 Feb 2017 21:34:13 +0100 Subject: [PATCH 9/9] info : use Archive.info() for both JSON and human display --- src/borg/archive.py | 9 +++++---- src/borg/archiver.py | 45 ++++++++++++++++++++++++-------------------- src/borg/helpers.py | 5 +++++ 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 7b87c0bdf..20746aa0f 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -28,7 +28,7 @@ from .helpers import Chunk, ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError from .helpers import uid2user, user2uid, gid2group, group2gid from .helpers import parse_timestamp, to_localtime -from .helpers import format_time, format_timedelta, format_file_size, file_status +from .helpers import format_time, format_timedelta, format_file_size, file_status, FileSize from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates from .helpers import StableDict from .helpers import bin_to_hex @@ -70,9 +70,9 @@ class Statistics: def as_dict(self): return { - 'original_size': self.osize, - 'compressed_size': self.csize, - 'deduplicated_size': self.usize, + 'original_size': FileSize(self.osize), + 'compressed_size': FileSize(self.csize), + 'deduplicated_size': FileSize(self.usize), 'nfiles': self.nfiles, } @@ -379,6 +379,7 @@ class Archive: 'command_line': self.metadata.cmdline, 'hostname': self.metadata.hostname, 'username': self.metadata.username, + 'comment': self.metadata.get('comment', ''), }) return info diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 39504dc02..8ecba523f 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -17,7 +17,7 @@ import textwrap import time import traceback from binascii import unhexlify -from datetime import datetime +from datetime import datetime, timedelta from itertools import zip_longest from .logger import create_logger, setup_logging @@ -37,7 +37,8 @@ from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from .helpers import Error, NoManifestError from .helpers import location_validator, archivename_validator, ChunkerParams, CompressionSpec from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS -from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter, format_time, format_file_size, format_archive +from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter +from .helpers import format_time, format_timedelta, format_file_size, format_archive from .helpers import safe_encode, remove_surrogates, bin_to_hex, prepare_dump_dict from .helpers import prune_within, prune_split from .helpers import to_localtime, timestamp @@ -52,7 +53,7 @@ from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm from .helpers import ErrorIgnoringTextIOWrapper from .helpers import ProgressIndicatorPercent -from .helpers import BorgJsonEncoder, basic_json_data, json_print +from .helpers import basic_json_data, json_print from .item import Item from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey from .keymanager import KeyManager @@ -997,25 +998,29 @@ class Archiver: for i, archive_name in enumerate(archive_names, 1): archive = Archive(repository, key, manifest, archive_name, cache=cache, consider_part_files=args.consider_part_files) + info = archive.info() if args.json: - output_data.append(archive.info()) + output_data.append(info) else: - stats = archive.calc_stats(cache) - print('Archive name: %s' % archive.name) - print('Archive fingerprint: %s' % archive.fpr) - print('Comment: %s' % archive.metadata.get('comment', '')) - print('Hostname: %s' % archive.metadata.hostname) - print('Username: %s' % archive.metadata.username) - print('Time (start): %s' % format_time(to_localtime(archive.ts))) - print('Time (end): %s' % format_time(to_localtime(archive.ts_end))) - print('Duration: %s' % archive.duration_from_meta) - print('Number of files: %d' % stats.nfiles) - print('Command line: %s' % format_cmdline(archive.metadata.cmdline)) - print('Utilization of max. archive size: %d%%' % (100 * cache.chunks[archive.id].csize / MAX_DATA_SIZE)) - print(DASHES) - print(STATS_HEADER) - print(str(stats)) - print(str(cache)) + info['duration'] = format_timedelta(timedelta(seconds=info['duration'])) + info['command_line'] = format_cmdline(info['command_line']) + print(textwrap.dedent(""" + Archive name: {name} + Archive fingerprint: {id} + Comment: {comment} + Hostname: {hostname} + Username: {username} + Time (start): {start} + Time (end): {end} + Duration: {duration} + Number of files: {stats[nfiles]} + Command line: {command_line} + Utilization of max. archive size: {limits[max_archive_size]:.0%} + ------------------------------------------------------------------------------ + Original size Compressed size Deduplicated size + This archive: {stats[original_size]:>20s} {stats[compressed_size]:>20s} {stats[deduplicated_size]:>20s} + {cache} + """).strip().format(cache=cache, **info)) if self.exit_code: break if not args.json and len(archive_names) - i: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 792e34571..09b15cc20 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -831,6 +831,11 @@ def format_file_size(v, precision=2, sign=False): return sizeof_fmt_decimal(v, suffix='B', sep=' ', precision=precision, sign=sign) +class FileSize(int): + def __format__(self, format_spec): + return format_file_size(int(self)).__format__(format_spec) + + def parse_file_size(s): """Return int from file size (1234, 55G, 1.7T).""" if not s: