From 7a1316cb793f3de08e516c20492165bfaa3a5fae Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 22 Nov 2015 19:53:44 +0100 Subject: [PATCH] implement ProgressIndicators, use it for repo check and segment replay, fixes #195, fixes #188 --- borg/archiver.py | 5 +-- borg/helpers.py | 76 ++++++++++++++++++++++++++++++++++++++++++++++ borg/repository.py | 23 ++++++++++++-- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index e88dc51fe..9e2d2eb5f 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -93,10 +93,7 @@ class Archiver: env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING', truish=('YES', )): return EXIT_ERROR if not args.archives_only: - logger.info('Starting repository check...') - if repository.check(repair=args.repair): - logger.info('Repository check complete, no problems found.') - else: + if not repository.check(repair=args.repair): return EXIT_WARNING if not args.repo_only and not ArchiveChecker().check( repository, repair=args.repair, archive=args.repository.archive, last=args.last): diff --git a/borg/helpers.py b/borg/helpers.py index 343bcc076..8516496c5 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -890,6 +890,82 @@ def yes(msg=None, retry_msg=None, false_msg=None, true_msg=None, ofile.flush() +class ProgressIndicatorPercent: + def __init__(self, total, step=5, start=0, same_line=False, msg="%3.0f%%", file=sys.stderr): + """ + Percentage-based progress indicator + + :param total: total amount of items + :param step: step size in percent + :param start: at which percent value to start + :param file: output file, default: sys.stderr + """ + self.counter = 0 # 0 .. (total-1) + self.total = total + self.trigger_at = start # output next percentage value when reaching (at least) this + self.step = step + self.file = file + self.msg = msg + self.same_line = same_line + + def progress(self, current=None): + if current is None: + current = self.counter + else: + self.counter = current + self.counter += 1 + pct = current * 100 / self.total + if pct >= self.trigger_at: + self.trigger_at += self.step + return pct + + def show(self, current=None): + pct = self.progress(current) + if pct is not None: + return self.output(pct) + + def output(self, percent): + print(self.msg % percent, file=self.file, end='\r' if self.same_line else '\n') + + def finish(self): + if self.same_line: + print(" " * len(self.msg % 100.0), file=self.file, end='\r') + + + +class ProgressIndicatorEndless: + def __init__(self, step=10, file=sys.stderr): + """ + Progress indicator (long row of dots) + + :param step: every Nth call, call the func + :param file: output file, default: sys.stderr + """ + self.counter = 0 # call counter + self.triggered = 0 # increases 1 per trigger event + self.step = step # trigger every calls + self.file = file + + def progress(self): + self.counter += 1 + trigger = self.counter % self.step == 0 + if trigger: + self.triggered += 1 + return trigger + + def show(self): + trigger = self.progress() + if trigger: + return self.output(self.triggered) + + def output(self, triggered): + print('.', end='', file=self.file) # python 3.3 gives us flush=True + self.file.flush() + + def finish(self): + print(file=self.file) + + def sysinfo(): info = [] info.append('Platform: %s' % (' '.join(platform.uname()), )) diff --git a/borg/repository.py b/borg/repository.py index f3189bc3b..49e3ac23d 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -10,7 +10,8 @@ import shutil import struct from zlib import crc32 -from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, unhexlify +from .helpers import Error, ErrorWithTraceback, IntegrityError, read_msgpack, write_msgpack, \ + unhexlify, ProgressIndicatorPercent from .hashindex import NSIndex from .locking import UpgradableLock, LockError, LockErrorT from .lrucache import LRUCache @@ -243,13 +244,17 @@ class Repository: def replay_segments(self, index_transaction_id, segments_transaction_id): self.prepare_txn(index_transaction_id, do_cleanup=False) try: - for segment, filename in self.io.segment_iterator(): + segment_count = sum(1 for _ in self.io.segment_iterator()) + pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%", same_line=True) + for i, (segment, filename) in enumerate(self.io.segment_iterator()): + pi.show(i) if index_transaction_id is not None and segment <= index_transaction_id: continue if segment > segments_transaction_id: break objects = self.io.iter_objects(segment) self._update_index(segment, objects) + pi.finish() self.write_index() finally: self.rollback() @@ -299,6 +304,7 @@ class Repository: error_found = True logger.error(msg) + logger.info('Starting repository check') assert not self._active_txn try: transaction_id = self.get_transaction_id() @@ -314,7 +320,10 @@ class Repository: self.io.cleanup(transaction_id) segments_transaction_id = self.io.get_segments_transaction_id() self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! - for segment, filename in self.io.segment_iterator(): + segment_count = sum(1 for _ in self.io.segment_iterator()) + pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.0f%%", same_line=True) + for i, (segment, filename) in enumerate(self.io.segment_iterator()): + pi.show(i) if segment > transaction_id: continue try: @@ -326,6 +335,7 @@ class Repository: self.io.recover_segment(segment, filename) objects = list(self.io.iter_objects(segment)) self._update_index(segment, objects, report_error) + pi.finish() # self.index, self.segments, self.compact now reflect the state of the segment files up to # We might need to add a commit tag if no committed segment is found if repair and segments_transaction_id is None: @@ -345,6 +355,13 @@ class Repository: self.compact_segments() self.write_index() self.rollback() + if error_found: + if repair: + logger.info('Completed repository check, errors found and repaired.') + else: + logger.info('Completed repository check, errors found.') + else: + logger.info('Completed repository check, no problems found.') return not error_found or repair def rollback(self):