From 3eebea8745cf71a1f87d8d988bdbaf6210ab5ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:12:04 -0400 Subject: [PATCH 01/27] explain what the --progress flag does exactly an alternative to this would be to use more than a letter in the output, for example: orig: 16.82 GB comp: 9.44 GB dedup: 25.86 MB home/ --- borg/archiver.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5a59fad18..72ad7f9f3 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -670,7 +670,10 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='print statistics for the created archive') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, - help='print progress while creating the archive') + help="""print progress while creating + the archive, showing Original, + Compressed and Deduplicated sizes, + followed by the path being processd""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') From 3827c2b1074470e5b693506c89bd41c213c49c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:13:59 -0400 Subject: [PATCH 02/27] also show the number of files processed in --progress --- borg/archiver.py | 7 +++---- borg/helpers.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 72ad7f9f3..5f1910ac9 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -670,10 +670,9 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='print statistics for the created archive') subparser.add_argument('-p', '--progress', dest='progress', action='store_true', default=False, - help="""print progress while creating - the archive, showing Original, - Compressed and Deduplicated sizes, - followed by the path being processd""") + help="""print progress while creating the archive, showing Original, + Compressed and Deduplicated sizes, followed by the Number of files seen + and the path being processd""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') diff --git a/borg/helpers.py b/borg/helpers.py index e3d6bc8cf..d0b5b7ad8 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -177,8 +177,8 @@ class Statistics: path = remove_surrogates(item[b'path']) if item else '' if len(path) > 43: path = '%s...%s' % (path[:20], path[-20:]) - msg = '%9s O %9s C %9s D %-43s' % ( - format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize), path) + msg = '%9s O %9s C %9s D %d N %-43s' % ( + format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize), self.nfiles, path) else: msg = ' ' * 79 print(msg, file=sys.stderr, end='\r') From b120e5f11965b4c69d6f52bb03b66c2e7f3dde8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:16:31 -0400 Subject: [PATCH 03/27] output more progress information without this, there would be a solid 20 seconds here without any sort of output on the console, regardless of the verbosity level. this makes nice incremental messages telling the user that borg is not stalled (or waiting for a lock, for that matter) the "processing files" message is a little clunky, as we somewhat abuse the cache to figure out if we are just starting... but it helps if there are problems reading the actual files: it tells us the initialization is basically complete and we're going ahead with the reading of all the files. --- borg/archive.py | 3 +++ borg/cache.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/borg/archive.py b/borg/archive.py index 2829769f2..ba06850c5 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -503,7 +503,10 @@ Number of files: {0.stats.nfiles} else: self.hard_links[st.st_ino, st.st_dev] = safe_path path_hash = self.key.id_hash(os.path.join(self.cwd, path).encode('utf-8', 'surrogateescape')) + first_run = not cache.files ids = cache.file_known_and_unchanged(path_hash, st) + if first_run: + logger.info('processing files') chunks = None if ids is not None: # Make sure all ids are available diff --git a/borg/cache.py b/borg/cache.py index e793f0f5d..3fab7cef2 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -48,6 +48,7 @@ class Cache: self.manifest = manifest self.path = path or os.path.join(get_cache_dir(), hexlify(repository.id).decode('ascii')) self.do_files = do_files + logger.info('initializing cache') # Warn user before sending data to a never seen before unencrypted repository if not os.path.exists(self.path): if warn_if_unencrypted and isinstance(key, PlaintextKey): @@ -69,6 +70,7 @@ class Cache: # Make sure an encrypted repository has not been swapped for an unencrypted repository if self.key_type is not None and self.key_type != str(key.TYPE): raise self.EncryptionMethodMismatch() + logger.info('synchronizing cache') self.sync() self.commit() @@ -163,6 +165,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""") def _read_files(self): self.files = {} self._newest_mtime = 0 + logger.info('reading files cache') with open(os.path.join(self.path, 'files'), 'rb') as fd: u = msgpack.Unpacker(use_list=True) while True: From d8f8076984ca79f550491879b8059044f7e9274a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:20:58 -0400 Subject: [PATCH 04/27] factor out status output so it is consistent as it was, surrogates were not always removed, for example we may also want to output at different levels or control if we want to print unchanged files and so on --- borg/archiver.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5f1910ac9..b46a2f789 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -56,6 +56,9 @@ class Archiver: msg = args and msg % args or msg logger.info(msg) + def print_status(self, status, path): + logger.info("%1s %s", status, remove_surrogates(path)) + def do_serve(self, args): """Start in server mode. This command is usually not used manually. """ @@ -143,7 +146,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") self.print_error('%s: %s', path, e) else: status = '-' - self.print_verbose("%1s %s", status, path) + self.print_status(status, path) continue path = os.path.normpath(path) if args.dontcross: @@ -238,7 +241,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") status = '-' # dry run, item was not backed up # output ALL the stuff - it can be easily filtered using grep. # even stuff considered unchanged might be interesting. - self.print_verbose("%1s %s", status, remove_surrogates(path)) + self.print_status(status, path) def do_extract(self, args): """Extract archive contents""" From e8cf28f9a07bfdd58a7559cc0c5473f67ac7e79e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:22:45 -0400 Subject: [PATCH 05/27] only show all files found in debug a single -v flag shouldn't flood the console with all the files in the path specified, it makes -v basically useless this way, -v can also be used with --progress to have nicer output: initializing cache reading files cache processing files 5.20 GB O 2.66 GB C 25.13 MB D 27576 N baz/... --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index b46a2f789..93703a5a2 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -57,7 +57,7 @@ class Archiver: logger.info(msg) def print_status(self, status, path): - logger.info("%1s %s", status, remove_surrogates(path)) + logger.debug("%1s %s", status, remove_surrogates(path)) def do_serve(self, args): """Start in server mode. This command is usually not used manually. From 7f28244cfe266fa76f6cdd29abc37b649bddefd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 21:38:43 -0400 Subject: [PATCH 06/27] fix conflict between --stats and --progress the --stats output would be slightly garbled by --progress, because of the \r that is output at the last line... example: initializing cache reading files cache processing files ------------------------------------------------------------------------------ s/twotone Archive name: 2015-10-15-test --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index d0b5b7ad8..987e35f8d 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -181,7 +181,7 @@ class Statistics: format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize), self.nfiles, path) else: msg = ' ' * 79 - print(msg, file=sys.stderr, end='\r') + print(msg, file=sys.stderr, end=final and "\n" or "\r") sys.stderr.flush() From ecae1630727948720b47ae135371cee2c86421cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 22:20:40 -0400 Subject: [PATCH 07/27] use all available columns for path in progress we use the new get_terminal_size() function, with a fallback for Python 3.2. we default to 80 columns. then we generate the stats bit and fill the rest with the path, as previously, but with a possibly larger field. note that this works with resizes in my test (uxterm) --- borg/helpers.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 987e35f8d..9650807c5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -8,6 +8,11 @@ import grp import os import pwd import re +try: + from shutil import get_terminal_size +except ImportError: + def get_terminal_size(fallback): + return (os.environ.get('COLUMNS', fallback[0]), os.environ.get('LINES', fallback[1])) import sys import time import unicodedata @@ -168,19 +173,24 @@ class Statistics: %-15s {0.osize:>20s} {0.csize:>20s} {0.usize:>20s}""") def __format__(self, format_spec): - fields = ['osize', 'csize', 'usize'] - FormattedStats = namedtuple('FormattedStats', fields) - return format_spec.format(FormattedStats(*map(format_file_size, [ getattr(self, x) for x in fields ]))) + sizes = ['osize', 'csize', 'usize'] + others = ['nfiles'] + fields = list(map(format_file_size, [ getattr(self, x) for x in sizes ])) + fields += [ getattr(self, x) for x in others ] + FormattedStats = namedtuple('FormattedStats', sizes + others) + return format_spec.format(FormattedStats(*fields)) def show_progress(self, item=None, final=False): + (columns, lines) = get_terminal_size((80, 24)) if not final: + msg = format(self, '{0.osize:9.9s} O {0.csize:9.9s} C {0.usize:9.9s} D {0.nfiles} N ') path = remove_surrogates(item[b'path']) if item else '' - if len(path) > 43: - path = '%s...%s' % (path[:20], path[-20:]) - msg = '%9s O %9s C %9s D %d N %-43s' % ( - format_file_size(self.osize), format_file_size(self.csize), format_file_size(self.usize), self.nfiles, path) + space = columns - len(msg) + if space < len('...') + len(path): + path = '%s...%s' % (path[:(space//2)-len('...')], path[-space//2:]) + msg += "{0:<{space}}".format(path, space=space) else: - msg = ' ' * 79 + msg = ' ' * columns print(msg, file=sys.stderr, end=final and "\n" or "\r") sys.stderr.flush() From 75c993b875dde3d46ff2ee28709b56dbadd09118 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 22:23:53 -0400 Subject: [PATCH 08/27] simplify progress display we stop enforcing a minimum width for fields, it changes only on logarithmic boundaries, so not a big problem. string conversion is implicit this gives us a little more width for the path --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index 9650807c5..2153fd227 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -183,7 +183,7 @@ class Statistics: def show_progress(self, item=None, final=False): (columns, lines) = get_terminal_size((80, 24)) if not final: - msg = format(self, '{0.osize:9.9s} O {0.csize:9.9s} C {0.usize:9.9s} D {0.nfiles} N ') + msg = format(self, '{0.osize} O {0.csize} C {0.usize} D {0.nfiles} N ') path = remove_surrogates(item[b'path']) if item else '' space = columns - len(msg) if space < len('...') + len(path): From 6b547e6554b0e5879d8d87bc5b6fc9b2e03a1418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Thu, 15 Oct 2015 22:26:02 -0400 Subject: [PATCH 09/27] no need to flush stderr in my tests --- borg/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index 2153fd227..e140833c4 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -192,7 +192,6 @@ class Statistics: else: msg = ' ' * columns print(msg, file=sys.stderr, end=final and "\n" or "\r") - sys.stderr.flush() def get_keys_dir(): From 118ee8679f56f23004f3920df932f6c0ae7cb241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 00:55:53 -0400 Subject: [PATCH 10/27] remove custom microformat from stats i saw the errors in my ways: __format__ is only to customize the "format mini-language", what comes after ":" in a new string format. unfortunately, we cannot easily refer to individual fields in there, short of re-implementing a new formatting language, which seems silly. instead, we use properties to extract human-readable versions of the statistics. more intuitive and certainly a more common pattern than the exotic __format__(). also add unit tests to prove this all works --- borg/helpers.py | 28 ++++++++++++++----------- borg/testsuite/helpers.py | 44 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index e140833c4..9c6a31fe5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -168,22 +168,26 @@ class Statistics: return buf def __str__(self): - return format(self, """\ + return """\ Original size Compressed size Deduplicated size -%-15s {0.osize:>20s} {0.csize:>20s} {0.usize:>20s}""") +%-15s {0.osize_fmt:>20s} {0.csize_fmt:>20s} {0.usize_fmt:>20s}""".format(self) - def __format__(self, format_spec): - sizes = ['osize', 'csize', 'usize'] - others = ['nfiles'] - fields = list(map(format_file_size, [ getattr(self, x) for x in sizes ])) - fields += [ getattr(self, x) for x in others ] - FormattedStats = namedtuple('FormattedStats', sizes + others) - return format_spec.format(FormattedStats(*fields)) + @property + def osize_fmt(self): + return format_file_size(self.osize) - def show_progress(self, item=None, final=False): + @property + def usize_fmt(self): + return format_file_size(self.usize) + + @property + def csize_fmt(self): + return format_file_size(self.csize) + + def show_progress(self, item=None, final=False, stream=None): (columns, lines) = get_terminal_size((80, 24)) if not final: - msg = format(self, '{0.osize} O {0.csize} C {0.usize} D {0.nfiles} N ') + msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self) path = remove_surrogates(item[b'path']) if item else '' space = columns - len(msg) if space < len('...') + len(path): @@ -191,7 +195,7 @@ class Statistics: msg += "{0:<{space}}".format(path, space=space) else: msg = ' ' * columns - print(msg, file=sys.stderr, end=final and "\n" or "\r") + print(msg, file=stream or sys.stderr, end=final and "\n" or "\r") def get_keys_dir(): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 620a77c14..00292c813 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -1,6 +1,9 @@ +import pdb + import hashlib from time import mktime, strptime from datetime import datetime, timezone, timedelta +from io import StringIO import os import pytest @@ -8,7 +11,7 @@ import sys import msgpack from ..helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, \ - prune_within, prune_split, get_cache_dir, \ + prune_within, prune_split, get_cache_dir, Statistics, \ StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams from . import BaseTestCase @@ -399,3 +402,42 @@ def test_get_cache_dir(): # reset old env if old_env is not None: os.environ['BORG_CACHE_DIR'] = old_env + +@pytest.fixture() +def stats(): + stats = Statistics() + stats.update(10, 10, unique=True) + return stats + +def test_stats_basic(stats): + assert stats.osize == stats.csize == stats.usize == 10 + stats.update(10, 10, unique=False) + assert stats.osize == stats.csize == 20 + assert stats.usize == 10 + +def tests_stats_progress(stats, columns = 80): + os.environ['COLUMNS'] = str(columns) + io = StringIO() + stats.show_progress(stream=io) + s = '10 B O 10 B C 10 B D 0 N ' + buf = ' ' * (columns - len(s)) + assert io.getvalue() == s + buf + "\r" + + io = StringIO() + stats.update(10**3, 0, unique=False) + stats.show_progress(item={b'path': 'foo'}, final=False, stream=io) + s = '1.01 kB O 10 B C 10 B D 0 N foo' + buf = ' ' * (columns - len(s)) + assert io.getvalue() == s + buf + "\r" + io = StringIO() + stats.show_progress(item={b'path': 'foo'*40}, final=False, stream=io) + s = '1.01 kB O 10 B C 10 B D 0 N foofoofoofoofoofoofoofo...oofoofoofoofoofoofoofoofoo' + buf = ' ' * (columns - len(s)) + assert io.getvalue() == s + buf + "\r" + +def test_stats_format(stats): + assert str(stats) == """\ + Original size Compressed size Deduplicated size +%-15s 10 B 10 B 10 B""" + s = "{0.osize_fmt}".format(stats) + assert s == "10 B" From 66bfc6fce86219dfde3f2024ea7c95cb48af6e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 01:13:47 -0400 Subject: [PATCH 11/27] get rid of print_() function this function was over-coupling the cache system and the statistics module. they are now almost decoupled insofar as the cache system has its own rendering system now that is called separately. furthermore, we have a much more flexible formatting system that is used coherently between --progress and --stats the degenerate case here is if we want to change the label in the statistics summary: in this case we need to override the default __str__() representation to insert our own label. --- borg/archiver.py | 12 ++++++++---- borg/helpers.py | 14 +++++--------- borg/testsuite/helpers.py | 3 ++- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 93703a5a2..f3530549f 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -167,7 +167,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") archive.end = datetime.now() print('-' * 78) print(str(archive)) - print(archive.stats.print_('This archive:', cache)) + print(str(archive.stats)) + print(str(cache)) print('-' * 78) return self.exit_code @@ -313,7 +314,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") repository.commit() cache.commit() if args.stats: - logger.info(stats.print_('Deleted data:', cache)) + logger.info(stats.summary.format(label='Deleted data:', stats=stats)) + logger.info(str(cache)) else: if not args.cache_only: print("You requested to completely DELETE the repository *including* all archives it contains:", file=sys.stderr) @@ -414,7 +416,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print('Time: %s' % to_localtime(archive.ts).strftime('%c')) print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) print('Number of files: %d' % stats.nfiles) - print(stats.print_('This archive:', cache)) + print(str(stats)) + print(str(cache)) return self.exit_code def do_prune(self, args): @@ -459,7 +462,8 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") repository.commit() cache.commit() if args.stats: - logger.info(stats.print_('Deleted data:', cache)) + logger.info(stats.summary.format(label='Deleted data:', stats=stats)) + logger.info(str(cache)) return self.exit_code def do_upgrade(self, args): diff --git a/borg/helpers.py b/borg/helpers.py index 9c6a31fe5..5a93bd182 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -161,16 +161,12 @@ class Statistics: if unique: self.usize += csize - def print_(self, label, cache): - buf = str(self) % label - buf += "\n" - buf += str(cache) - return buf - - def __str__(self): - return """\ + summary = """\ Original size Compressed size Deduplicated size -%-15s {0.osize_fmt:>20s} {0.csize_fmt:>20s} {0.usize_fmt:>20s}""".format(self) +{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s} +""" + def __str__(self): + return self.summary.format(stats=self, label='This archive:') @property def osize_fmt(self): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 00292c813..37f6fcc4c 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -438,6 +438,7 @@ def tests_stats_progress(stats, columns = 80): def test_stats_format(stats): assert str(stats) == """\ Original size Compressed size Deduplicated size -%-15s 10 B 10 B 10 B""" +This archive: 10 B 10 B 10 B +""" s = "{0.osize_fmt}".format(stats) assert s == "10 B" From 28d0a9ce843e3461b45fbbfd1c8cc1a2b84ef82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 01:38:53 -0400 Subject: [PATCH 12/27] human-readable representation of stats --- borg/helpers.py | 6 ++++++ borg/testsuite/helpers.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/borg/helpers.py b/borg/helpers.py index 5a93bd182..b793ca492 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -168,6 +168,12 @@ class Statistics: def __str__(self): return self.summary.format(stats=self, label='This archive:') + def __repr__(self): + fmt = "<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>" + return fmt.format(cls=type(self).__name__, + hash=id(self), + self=self) + @property def osize_fmt(self): return format_file_size(self.osize) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 37f6fcc4c..ac2cbacad 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -442,3 +442,5 @@ This archive: 10 B 10 B 10 B """ s = "{0.osize_fmt}".format(stats) assert s == "10 B" + # kind of redundant, but id is variable so we can't match reliably + assert stats.__repr__() == ''.format(id(stats)) From d666c86bfc96e56e09248ce21f460342c992c6e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 01:54:04 -0400 Subject: [PATCH 13/27] adjust display of --stats it was broken by recent commits. also remove the __format__() anti-pattern from cache as well. --- borg/archive.py | 6 ++---- borg/cache.py | 10 +++++----- borg/helpers.py | 3 +-- borg/testsuite/helpers.py | 3 +-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index ba06850c5..7c0d56de1 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -201,14 +201,12 @@ class Archive: return format_timedelta(self.end-self.start) def __str__(self): - buf = '''Archive name: {0.name} + return '''Archive name: {0.name} Archive fingerprint: {0.fpr} Start time: {0.start:%c} End time: {0.end:%c} Duration: {0.duration} -Number of files: {0.stats.nfiles} -{0.cache}'''.format(self) - return buf +Number of files: {0.stats.nfiles}'''.format(self) def __repr__(self): return 'Archive(%r)' % self.name diff --git a/borg/cache.py b/borg/cache.py index 3fab7cef2..729149c21 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -78,20 +78,20 @@ class Cache: self.close() def __str__(self): - return format(self, """\ + fmt = """\ All archives: {0.total_size:>20s} {0.total_csize:>20s} {0.unique_csize:>20s} Unique chunks Total chunks -Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""") +Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" + return fmt.format(self.format_tuple()) - def __format__(self, format_spec): + def format_tuple(self): # XXX: this should really be moved down to `hashindex.pyx` Summary = namedtuple('Summary', ['total_size', 'total_csize', 'unique_size', 'unique_csize', 'total_unique_chunks', 'total_chunks']) stats = Summary(*self.chunks.summarize())._asdict() for field in ['total_size', 'total_csize', 'unique_csize']: stats[field] = format_file_size(stats[field]) - stats = Summary(**stats) - return format_spec.format(stats) + return Summary(**stats) def _confirm(self, message, env_var_override=None): print(message, file=sys.stderr) diff --git a/borg/helpers.py b/borg/helpers.py index b793ca492..a60776e12 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -163,8 +163,7 @@ class Statistics: summary = """\ Original size Compressed size Deduplicated size -{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s} -""" +{label:15} {stats.osize_fmt:>20s} {stats.csize_fmt:>20s} {stats.usize_fmt:>20s}""" def __str__(self): return self.summary.format(stats=self, label='This archive:') diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index ac2cbacad..577b08430 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -438,8 +438,7 @@ def tests_stats_progress(stats, columns = 80): def test_stats_format(stats): assert str(stats) == """\ Original size Compressed size Deduplicated size -This archive: 10 B 10 B 10 B -""" +This archive: 10 B 10 B 10 B""" s = "{0.osize_fmt}".format(stats) assert s == "10 B" # kind of redundant, but id is variable so we can't match reliably From 0d8525ad8f5792b1865b0fc459faa23595e2f138 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 02:01:56 -0400 Subject: [PATCH 14/27] add missing lines --- borg/archiver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index f3530549f..64e37d1cd 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -167,6 +167,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") archive.end = datetime.now() print('-' * 78) print(str(archive)) + print() print(str(archive.stats)) print(str(cache)) print('-' * 78) @@ -416,6 +417,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") print('Time: %s' % to_localtime(archive.ts).strftime('%c')) print('Command line:', remove_surrogates(' '.join(archive.metadata[b'cmdline']))) print('Number of files: %d' % stats.nfiles) + print() print(str(stats)) print(str(cache)) return self.exit_code From ce1aaa9dfa95625f1b5d98a988a54d7d051ff2cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 11:19:27 -0400 Subject: [PATCH 15/27] Revert "fix conflict between --stats and --progress" the columns handling fixed that isssue more elegantly This reverts commit 7f777784194b2625272a368c85c0712eb01df2fb. Conflicts: borg/helpers.py --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index a60776e12..a08bbef19 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -196,7 +196,7 @@ class Statistics: msg += "{0:<{space}}".format(path, space=space) else: msg = ' ' * columns - print(msg, file=stream or sys.stderr, end=final and "\n" or "\r") + print(msg, file=stream or sys.stderr, end="\r") def get_keys_dir(): From e4f325182ed60827e38b3c9e23a72272e46a60a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 11:21:19 -0400 Subject: [PATCH 16/27] Revert "no need to flush stderr in my tests" It is actually necessary, for now. This reverts commit 8fdd1eddf91a3d5dbc013ba6e4a170f8613598c0. Conflicts: borg/helpers.py --- borg/helpers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/borg/helpers.py b/borg/helpers.py index a08bbef19..1cad9ec87 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -197,6 +197,7 @@ class Statistics: else: msg = ' ' * columns print(msg, file=stream or sys.stderr, end="\r") + (stream or sys.stderr).flush() def get_keys_dir(): From 92ac120fb060dc50a9875ae59718703a40aa1ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Fri, 16 Oct 2015 11:30:41 -0400 Subject: [PATCH 17/27] default progress display to true if on a tty this makes --progress a toggle: if there's a terminal, it turns it off, if there isn't, it forces it on. --- borg/archiver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 64e37d1cd..f1c9f8c00 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -677,11 +677,11 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the created archive') - subparser.add_argument('-p', '--progress', dest='progress', - action='store_true', default=False, - help="""print progress while creating the archive, showing Original, + subparser.add_argument('-p', '--progress', dest='progress', const=not sys.stdin.isatty(), + action='store_const', default=sys.stdin.isatty(), + help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen - and the path being processd""") + and the path being processd, default: %(default)s""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') From 1c0fb82b59b44389b1bdb310e5bbbe889901950b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 18 Oct 2015 20:54:39 -0400 Subject: [PATCH 18/27] check for the stream --progress uses, not stdin --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index f1c9f8c00..a1eacd9e3 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -677,7 +677,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") subparser.add_argument('-s', '--stats', dest='stats', action='store_true', default=False, help='print statistics for the created archive') - subparser.add_argument('-p', '--progress', dest='progress', const=not sys.stdin.isatty(), + subparser.add_argument('-p', '--progress', dest='progress', const=not sys.stderr.isatty(), action='store_const', default=sys.stdin.isatty(), help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen From 4c6915cce56f56064bb8568ba5412bd8b91e569d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 18 Oct 2015 21:06:37 -0400 Subject: [PATCH 19/27] fix typos and remove debug code --- borg/archiver.py | 2 +- borg/testsuite/helpers.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index a1eacd9e3..7a50d1145 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -681,7 +681,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") action='store_const', default=sys.stdin.isatty(), help="""toggle progress display while creating the archive, showing Original, Compressed and Deduplicated sizes, followed by the Number of files seen - and the path being processd, default: %(default)s""") + and the path being processed, default: %(default)s""") subparser.add_argument('-e', '--exclude', dest='excludes', type=ExcludePattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 577b08430..c8f657fb5 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -1,5 +1,3 @@ -import pdb - import hashlib from time import mktime, strptime from datetime import datetime, timezone, timedelta @@ -442,4 +440,4 @@ This archive: 10 B 10 B 10 B"" s = "{0.osize_fmt}".format(stats) assert s == "10 B" # kind of redundant, but id is variable so we can't match reliably - assert stats.__repr__() == ''.format(id(stats)) + assert repr(stats) == ''.format(id(stats)) From 158e6b529afde1053ae27e92151d4dae68b44faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 18 Oct 2015 21:06:56 -0400 Subject: [PATCH 20/27] cosmetic: pep8 and io is actually just out --- borg/testsuite/helpers.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index c8f657fb5..1a83491df 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -413,25 +413,25 @@ def test_stats_basic(stats): assert stats.osize == stats.csize == 20 assert stats.usize == 10 -def tests_stats_progress(stats, columns = 80): +def tests_stats_progress(stats, columns=80): os.environ['COLUMNS'] = str(columns) - io = StringIO() - stats.show_progress(stream=io) + out = StringIO() + stats.show_progress(stream=out) s = '10 B O 10 B C 10 B D 0 N ' buf = ' ' * (columns - len(s)) - assert io.getvalue() == s + buf + "\r" + assert out.getvalue() == s + buf + "\r" - io = StringIO() + out = StringIO() stats.update(10**3, 0, unique=False) - stats.show_progress(item={b'path': 'foo'}, final=False, stream=io) + stats.show_progress(item={b'path': 'foo'}, final=False, stream=out) s = '1.01 kB O 10 B C 10 B D 0 N foo' buf = ' ' * (columns - len(s)) - assert io.getvalue() == s + buf + "\r" - io = StringIO() - stats.show_progress(item={b'path': 'foo'*40}, final=False, stream=io) + assert out.getvalue() == s + buf + "\r" + out = StringIO() + stats.show_progress(item={b'path': 'foo'*40}, final=False, stream=out) s = '1.01 kB O 10 B C 10 B D 0 N foofoofoofoofoofoofoofo...oofoofoofoofoofoofoofoofoo' buf = ' ' * (columns - len(s)) - assert io.getvalue() == s + buf + "\r" + assert out.getvalue() == s + buf + "\r" def test_stats_format(stats): assert str(stats) == """\ From f48bbc372583b1e44d4076c90862b64cc04c73bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 18 Oct 2015 21:07:28 -0400 Subject: [PATCH 21/27] move defaults up in alternate implementation, use nameduple --- borg/helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 1cad9ec87..fdfd2a31b 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -11,8 +11,9 @@ import re try: from shutil import get_terminal_size except ImportError: - def get_terminal_size(fallback): - return (os.environ.get('COLUMNS', fallback[0]), os.environ.get('LINES', fallback[1])) + def get_terminal_size(fallback=(80, 24)): + TerminalSize = namedtuple(TerminalSize, ['columns', 'lines']) + return TerminalSize(os.environ.get('COLUMNS', int(fallback[0])), os.environ.get('LINES', int(fallback[1]))) import sys import time import unicodedata @@ -186,7 +187,7 @@ class Statistics: return format_file_size(self.csize) def show_progress(self, item=None, final=False, stream=None): - (columns, lines) = get_terminal_size((80, 24)) + columns, lines = get_terminal_size() if not final: msg = '{0.osize_fmt} O {0.csize_fmt} C {0.usize_fmt} D {0.nfiles} N '.format(self) path = remove_surrogates(item[b'path']) if item else '' From 98512736e5b9e07ccaf8e158e697d29dbf31ac88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Sun, 18 Oct 2015 21:13:01 -0400 Subject: [PATCH 22/27] stats tests: vary values to possible catch errors --- borg/testsuite/helpers.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 1a83491df..9f7400bcd 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -404,40 +404,42 @@ def test_get_cache_dir(): @pytest.fixture() def stats(): stats = Statistics() - stats.update(10, 10, unique=True) + stats.update(20, 10, unique=True) return stats def test_stats_basic(stats): - assert stats.osize == stats.csize == stats.usize == 10 - stats.update(10, 10, unique=False) - assert stats.osize == stats.csize == 20 + assert stats.osize == 20 + assert stats.csize == stats.usize == 10 + stats.update(20, 10, unique=False) + assert stats.osize == 40 + assert stats.csize == 20 assert stats.usize == 10 def tests_stats_progress(stats, columns=80): os.environ['COLUMNS'] = str(columns) out = StringIO() stats.show_progress(stream=out) - s = '10 B O 10 B C 10 B D 0 N ' + s = '20 B O 10 B C 10 B D 0 N ' buf = ' ' * (columns - len(s)) assert out.getvalue() == s + buf + "\r" out = StringIO() stats.update(10**3, 0, unique=False) stats.show_progress(item={b'path': 'foo'}, final=False, stream=out) - s = '1.01 kB O 10 B C 10 B D 0 N foo' + s = '1.02 kB O 10 B C 10 B D 0 N foo' buf = ' ' * (columns - len(s)) assert out.getvalue() == s + buf + "\r" out = StringIO() stats.show_progress(item={b'path': 'foo'*40}, final=False, stream=out) - s = '1.01 kB O 10 B C 10 B D 0 N foofoofoofoofoofoofoofo...oofoofoofoofoofoofoofoofoo' + s = '1.02 kB O 10 B C 10 B D 0 N foofoofoofoofoofoofoofo...oofoofoofoofoofoofoofoofoo' buf = ' ' * (columns - len(s)) assert out.getvalue() == s + buf + "\r" def test_stats_format(stats): assert str(stats) == """\ Original size Compressed size Deduplicated size -This archive: 10 B 10 B 10 B""" +This archive: 20 B 10 B 10 B""" s = "{0.osize_fmt}".format(stats) - assert s == "10 B" + assert s == "20 B" # kind of redundant, but id is variable so we can't match reliably - assert repr(stats) == ''.format(id(stats)) + assert repr(stats) == ''.format(id(stats)) From 49d7c240a0ad37058eea7720421a1205b7d94391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 19 Oct 2015 09:10:04 -0400 Subject: [PATCH 23/27] cosmetic: don't word-wrap needlessly --- borg/helpers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index fdfd2a31b..2a04c4547 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -169,10 +169,7 @@ class Statistics: return self.summary.format(stats=self, label='This archive:') def __repr__(self): - fmt = "<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>" - return fmt.format(cls=type(self).__name__, - hash=id(self), - self=self) + return "<{cls} object at {hash:#x} ({self.osize}, {self.csize}, {self.usize})>".format(cls=type(self).__name__, hash=id(self), self=self) @property def osize_fmt(self): From f9bccd19870d1af47d2ccb423244452aaf2e8ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 19 Oct 2015 09:10:19 -0400 Subject: [PATCH 24/27] revert policy decision of showing all files listing only on debug --- borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 7a50d1145..366d08891 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -57,7 +57,7 @@ class Archiver: logger.info(msg) def print_status(self, status, path): - logger.debug("%1s %s", status, remove_surrogates(path)) + logger.info("%1s %s", status, remove_surrogates(path)) def do_serve(self, args): """Start in server mode. This command is usually not used manually. From 12d22d5940b7bdfa1ac1a55cc06d0a1f256a9916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 19 Oct 2015 09:10:45 -0400 Subject: [PATCH 25/27] try to fix build on py32 --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index 2a04c4547..ab86f45f9 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -12,7 +12,7 @@ try: from shutil import get_terminal_size except ImportError: def get_terminal_size(fallback=(80, 24)): - TerminalSize = namedtuple(TerminalSize, ['columns', 'lines']) + TerminalSize = namedtuple('TerminalSize', ['columns', 'lines']) return TerminalSize(os.environ.get('COLUMNS', int(fallback[0])), os.environ.get('LINES', int(fallback[1]))) import sys import time From a3084b7174932533c89b8df7294995eca1ac390e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 19 Oct 2015 14:41:02 -0400 Subject: [PATCH 26/27] try to fix build on py32 again --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index ab86f45f9..d262d9d51 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -13,7 +13,7 @@ try: except ImportError: def get_terminal_size(fallback=(80, 24)): TerminalSize = namedtuple('TerminalSize', ['columns', 'lines']) - return TerminalSize(os.environ.get('COLUMNS', int(fallback[0])), os.environ.get('LINES', int(fallback[1]))) + return TerminalSize(int(os.environ.get('COLUMNS', int(fallback[0]))), int(os.environ.get('LINES', int(fallback[1])))) import sys import time import unicodedata From 38bde2667387f951effcc498a0bff38da2755d28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Mon, 19 Oct 2015 15:47:05 -0400 Subject: [PATCH 27/27] remove needless double type conversion i got confused and thought i was setting the environment (in which case you need to cast to str anyways, so this was wrong even then) --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index d262d9d51..e22a5d0c1 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -13,7 +13,7 @@ try: except ImportError: def get_terminal_size(fallback=(80, 24)): TerminalSize = namedtuple('TerminalSize', ['columns', 'lines']) - return TerminalSize(int(os.environ.get('COLUMNS', int(fallback[0]))), int(os.environ.get('LINES', int(fallback[1])))) + return TerminalSize(int(os.environ.get('COLUMNS', fallback[0])), int(os.environ.get('LINES', fallback[1]))) import sys import time import unicodedata