1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-03-15 00:21:56 +00:00

Merge branch 'ThomasWaldmann-silent'

This commit is contained in:
Thomas Waldmann 2015-12-08 10:03:22 +01:00
commit f2d41b1a69
7 changed files with 101 additions and 55 deletions

View file

@ -3,7 +3,6 @@ from datetime import datetime
from getpass import getuser from getpass import getuser
from itertools import groupby from itertools import groupby
import errno import errno
import logging
from .logger import create_logger from .logger import create_logger
logger = create_logger() logger = create_logger()
@ -663,20 +662,24 @@ class ArchiveChecker:
self.possibly_superseded = set() self.possibly_superseded = set()
def check(self, repository, repair=False, archive=None, last=None): def check(self, repository, repair=False, archive=None, last=None):
self.report_progress('Starting archive consistency check...') logger.info('Starting archive consistency check...')
self.check_all = archive is None and last is None self.check_all = archive is None and last is None
self.repair = repair self.repair = repair
self.repository = repository self.repository = repository
self.init_chunks() self.init_chunks()
self.key = self.identify_key(repository) self.key = self.identify_key(repository)
if Manifest.MANIFEST_ID not in self.chunks: if Manifest.MANIFEST_ID not in self.chunks:
logger.error("Repository manifest not found!")
self.error_found = True
self.manifest = self.rebuild_manifest() self.manifest = self.rebuild_manifest()
else: else:
self.manifest, _ = Manifest.load(repository, key=self.key) self.manifest, _ = Manifest.load(repository, key=self.key)
self.rebuild_refcounts(archive=archive, last=last) self.rebuild_refcounts(archive=archive, last=last)
self.orphan_chunks_check() self.orphan_chunks_check()
self.finish() self.finish()
if not self.error_found: if self.error_found:
logger.error('Archive consistency check complete, problems found.')
else:
logger.info('Archive consistency check complete, no problems found.') logger.info('Archive consistency check complete, no problems found.')
return self.repair or not self.error_found return self.repair or not self.error_found
@ -696,11 +699,6 @@ class ArchiveChecker:
for id_ in result: for id_ in result:
self.chunks[id_] = (0, 0, 0) self.chunks[id_] = (0, 0, 0)
def report_progress(self, msg, error=False):
if error:
self.error_found = True
logger.log(logging.ERROR if error else logging.WARNING, msg)
def identify_key(self, repository): def identify_key(self, repository):
cdata = repository.get(next(self.chunks.iteritems())[0]) cdata = repository.get(next(self.chunks.iteritems())[0])
return key_factory(repository, cdata) return key_factory(repository, cdata)
@ -710,7 +708,7 @@ class ArchiveChecker:
Iterates through all objects in the repository looking for archive metadata blocks. Iterates through all objects in the repository looking for archive metadata blocks.
""" """
self.report_progress('Rebuilding missing manifest, this might take some time...', error=True) logger.info('Rebuilding missing manifest, this might take some time...')
manifest = Manifest(self.key, self.repository) manifest = Manifest(self.key, self.repository)
for chunk_id, _ in self.chunks.iteritems(): for chunk_id, _ in self.chunks.iteritems():
cdata = self.repository.get(chunk_id) cdata = self.repository.get(chunk_id)
@ -727,9 +725,9 @@ class ArchiveChecker:
except (TypeError, ValueError, StopIteration): except (TypeError, ValueError, StopIteration):
continue continue
if isinstance(archive, dict) and b'items' in archive and b'cmdline' in archive: if isinstance(archive, dict) and b'items' in archive and b'cmdline' in archive:
self.report_progress('Found archive ' + archive[b'name'].decode('utf-8'), error=True) logger.info('Found archive %s', archive[b'name'].decode('utf-8'))
manifest.archives[archive[b'name'].decode('utf-8')] = {b'id': chunk_id, b'time': archive[b'time']} manifest.archives[archive[b'name'].decode('utf-8')] = {b'id': chunk_id, b'time': archive[b'time']}
self.report_progress('Manifest rebuild complete', error=True) logger.info('Manifest rebuild complete.')
return manifest return manifest
def rebuild_refcounts(self, archive=None, last=None): def rebuild_refcounts(self, archive=None, last=None):
@ -771,7 +769,8 @@ class ArchiveChecker:
for chunk_id, size, csize in item[b'chunks']: for chunk_id, size, csize in item[b'chunks']:
if chunk_id not in self.chunks: if chunk_id not in self.chunks:
# If a file chunk is missing, create an all empty replacement chunk # If a file chunk is missing, create an all empty replacement chunk
self.report_progress('{}: Missing file chunk detected (Byte {}-{})'.format(item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size), error=True) logger.error('{}: Missing file chunk detected (Byte {}-{})'.format(item[b'path'].decode('utf-8', 'surrogateescape'), offset, offset + size))
self.error_found = True
data = bytes(size) data = bytes(size)
chunk_id = self.key.id_hash(data) chunk_id = self.key.id_hash(data)
cdata = self.key.encrypt(data) cdata = self.key.encrypt(data)
@ -800,7 +799,8 @@ class ArchiveChecker:
def report(msg, chunk_id, chunk_no): def report(msg, chunk_id, chunk_no):
cid = hexlify(chunk_id).decode('ascii') cid = hexlify(chunk_id).decode('ascii')
msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items msg += ' [chunk: %06d_%s]' % (chunk_no, cid) # see debug-dump-archive-items
self.report_progress(msg, error=True) self.error_found = True
logger.error(msg)
i = 0 i = 0
for state, items in groupby(archive[b'items'], missing_chunk_detector): for state, items in groupby(archive[b'items'], missing_chunk_detector):
@ -841,7 +841,8 @@ class ArchiveChecker:
logger.info('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives)) logger.info('Analyzing archive {} ({}/{})'.format(name, num_archives - i, num_archives))
archive_id = info[b'id'] archive_id = info[b'id']
if archive_id not in self.chunks: if archive_id not in self.chunks:
self.report_progress('Archive metadata block is missing', error=True) logger.error('Archive metadata block is missing!')
self.error_found = True
del self.manifest.archives[name] del self.manifest.archives[name]
continue continue
mark_as_possibly_superseded(archive_id) mark_as_possibly_superseded(archive_id)
@ -876,12 +877,13 @@ class ArchiveChecker:
unused.add(id_) unused.add(id_)
orphaned = unused - self.possibly_superseded orphaned = unused - self.possibly_superseded
if orphaned: if orphaned:
self.report_progress('{} orphaned objects found'.format(len(orphaned)), error=True) logger.error('{} orphaned objects found!'.format(len(orphaned)))
self.error_found = True
if self.repair: if self.repair:
for id_ in unused: for id_ in unused:
self.repository.delete(id_) self.repository.delete(id_)
else: else:
self.report_progress('Orphaned objects check skipped (needs all archives checked)') logger.warning('Orphaned objects check skipped (needs all archives checked).')
def finish(self): def finish(self):
if self.repair: if self.repair:

View file

@ -51,9 +51,8 @@ class ToggleAction(argparse.Action):
class Archiver: class Archiver:
def __init__(self, verbose=False, lock_wait=None): def __init__(self, lock_wait=None):
self.exit_code = EXIT_SUCCESS self.exit_code = EXIT_SUCCESS
self.verbose = verbose
self.lock_wait = lock_wait self.lock_wait = lock_wait
def open_repository(self, location, create=False, exclusive=False, lock=True): def open_repository(self, location, create=False, exclusive=False, lock=True):
@ -74,10 +73,9 @@ class Archiver:
self.exit_code = EXIT_WARNING # we do not terminate here, so it is a warning self.exit_code = EXIT_WARNING # we do not terminate here, so it is a warning
logger.warning(msg) logger.warning(msg)
def print_verbose(self, msg, *args): def print_file_status(self, status, path):
if self.verbose: if self.output_filter is None or status in self.output_filter:
msg = args and msg % args or msg logger.info("%1s %s", status, remove_surrogates(path))
logger.info(msg)
def do_serve(self, args): def do_serve(self, args):
"""Start in server mode. This command is usually not used manually. """Start in server mode. This command is usually not used manually.
@ -123,6 +121,7 @@ class Archiver:
def do_create(self, args): def do_create(self, args):
"""Create new archive""" """Create new archive"""
self.output_filter = args.output_filter
dry_run = args.dry_run dry_run = args.dry_run
t0 = datetime.now() t0 = datetime.now()
if not dry_run: if not dry_run:
@ -163,7 +162,7 @@ class Archiver:
self.print_warning('%s: %s', path, e) self.print_warning('%s: %s', path, e)
else: else:
status = '-' status = '-'
self.print_verbose("%1s %s", status, remove_surrogates(path)) self.print_file_status(status, path)
continue continue
path = os.path.normpath(path) path = os.path.normpath(path)
if args.one_file_system: if args.one_file_system:
@ -262,9 +261,7 @@ class Archiver:
status = '?' # need to add a status code somewhere status = '?' # need to add a status code somewhere
else: else:
status = '-' # dry run, item was not backed up status = '-' # dry run, item was not backed up
# output ALL the stuff - it can be easily filtered using grep. self.print_file_status(status, path)
# even stuff considered unchanged might be interesting.
self.print_verbose("%1s %s", status, remove_surrogates(path))
def do_extract(self, args): def do_extract(self, args):
"""Extract archive contents""" """Extract archive contents"""
@ -292,7 +289,7 @@ class Archiver:
if not args.dry_run: if not args.dry_run:
while dirs and not item[b'path'].startswith(dirs[-1][b'path']): while dirs and not item[b'path'].startswith(dirs[-1][b'path']):
archive.extract_item(dirs.pop(-1), stdout=stdout) archive.extract_item(dirs.pop(-1), stdout=stdout)
self.print_verbose(remove_surrogates(orig_path)) logger.info(remove_surrogates(orig_path))
try: try:
if dry_run: if dry_run:
archive.extract_item(item, dry_run=True) archive.extract_item(item, dry_run=True)
@ -378,7 +375,7 @@ class Archiver:
else: else:
archive = None archive = None
operations = FuseOperations(key, repository, manifest, archive) operations = FuseOperations(key, repository, manifest, archive)
self.print_verbose("Mounting filesystem") logger.info("Mounting filesystem")
try: try:
operations.mount(args.mountpoint, args.options, args.foreground) operations.mount(args.mountpoint, args.options, args.foreground)
except RuntimeError: except RuntimeError:
@ -481,12 +478,12 @@ class Archiver:
to_delete = [a for a in archives if a not in keep] to_delete = [a for a in archives if a not in keep]
stats = Statistics() stats = Statistics()
for archive in keep: for archive in keep:
self.print_verbose('Keeping archive: %s' % format_archive(archive)) logger.info('Keeping archive: %s' % format_archive(archive))
for archive in to_delete: for archive in to_delete:
if args.dry_run: if args.dry_run:
self.print_verbose('Would prune: %s' % format_archive(archive)) logger.info('Would prune: %s' % format_archive(archive))
else: else:
self.print_verbose('Pruning archive: %s' % format_archive(archive)) logger.info('Pruning archive: %s' % format_archive(archive))
Archive(repository, key, manifest, archive.name, cache).delete(stats) Archive(repository, key, manifest, archive.name, cache).delete(stats)
if to_delete and not args.dry_run: if to_delete and not args.dry_run:
manifest.write() manifest.write()
@ -663,11 +660,12 @@ class Archiver:
def build_parser(self, args=None, prog=None): def build_parser(self, args=None, prog=None):
common_parser = argparse.ArgumentParser(add_help=False, prog=prog) common_parser = argparse.ArgumentParser(add_help=False, prog=prog)
common_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=False, common_parser.add_argument('-v', '--verbose', '--info', dest='log_level',
help='verbose output') action='store_const', const='info', default='warning',
common_parser.add_argument('--log-level', dest='log_level', default='info', metavar='LEVEL', help='enable informative (verbose) output, work on log level INFO')
choices=('debug', 'info', 'warning', 'error', 'critical'), common_parser.add_argument('--debug', dest='log_level',
help='set the log level to LEVEL, default: %(default)s)') action='store_const', const='debug', default='warning',
help='enable debug output, work on log level DEBUG')
common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1, common_parser.add_argument('--lock-wait', dest='lock_wait', type=int, metavar='N', default=1,
help='wait for the lock, but max. N seconds (default: %(default)d).') help='wait for the lock, but max. N seconds (default: %(default)d).')
common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False, common_parser.add_argument('--show-rc', dest='show_rc', action='store_true', default=False,
@ -802,6 +800,8 @@ class Archiver:
help="""toggle progress display while creating the archive, showing Original, help="""toggle progress display while creating the archive, showing Original,
Compressed and Deduplicated sizes, followed by the Number of files seen Compressed and Deduplicated sizes, followed by the Number of files seen
and the path being processed, default: %(default)s""") and the path being processed, default: %(default)s""")
subparser.add_argument('--filter', dest='output_filter', metavar='STATUSCHARS',
help='only display items with the given status characters')
subparser.add_argument('-e', '--exclude', dest='excludes', subparser.add_argument('-e', '--exclude', dest='excludes',
type=ExcludePattern, action='append', type=ExcludePattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN') metavar="PATTERN", help='exclude paths matching PATTERN')
@ -1177,7 +1177,6 @@ class Archiver:
def run(self, args): def run(self, args):
os.umask(args.umask) # early, before opening files os.umask(args.umask) # early, before opening files
self.verbose = args.verbose
self.lock_wait = args.lock_wait self.lock_wait = args.lock_wait
RemoteRepository.remote_path = args.remote_path RemoteRepository.remote_path = args.remote_path
RemoteRepository.umask = args.umask RemoteRepository.umask = args.umask

View file

@ -365,7 +365,7 @@ class Repository:
if repair: if repair:
logger.info('Completed repository check, errors found and repaired.') logger.info('Completed repository check, errors found and repaired.')
else: else:
logger.info('Completed repository check, errors found.') logger.error('Completed repository check, errors found.')
else: else:
logger.info('Completed repository check, no problems found.') logger.info('Completed repository check, no problems found.')
return not error_found or repair return not error_found or repair

View file

@ -713,16 +713,40 @@ class ArchiverTestCase(ArchiverTestCaseBase):
os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago os.utime('input/file1', (now - 5, now - 5)) # 5 seconds ago
self.create_regular_file('file2', size=1024 * 80) self.create_regular_file('file2', size=1024 * 80)
self.cmd('init', self.repository_location) self.cmd('init', self.repository_location)
output = self.cmd('create', '--verbose', self.repository_location + '::test', 'input') output = self.cmd('create', '-v', self.repository_location + '::test', 'input')
self.assert_in("A input/file1", output) self.assert_in("A input/file1", output)
self.assert_in("A input/file2", output) self.assert_in("A input/file2", output)
# should find first file as unmodified # should find first file as unmodified
output = self.cmd('create', '--verbose', self.repository_location + '::test1', 'input') output = self.cmd('create', '-v', self.repository_location + '::test1', 'input')
self.assert_in("U input/file1", output) self.assert_in("U input/file1", output)
# this is expected, although surprising, for why, see: # this is expected, although surprising, for why, see:
# http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file # http://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file
self.assert_in("A input/file2", output) self.assert_in("A input/file2", output)
def test_create_topical(self):
now = time.time()
self.create_regular_file('file1', size=1024 * 80)
os.utime('input/file1', (now-5, now-5))
self.create_regular_file('file2', size=1024 * 80)
self.cmd('init', self.repository_location)
# no listing by default
output = self.cmd('create', self.repository_location + '::test', 'input')
self.assert_not_in('file1', output)
# shouldn't be listed even if unchanged
output = self.cmd('create', self.repository_location + '::test0', 'input')
self.assert_not_in('file1', output)
# should list the file as unchanged
output = self.cmd('create', '-v', '--filter=U', self.repository_location + '::test1', 'input')
self.assert_in('file1', output)
# should *not* list the file as changed
output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test2', 'input')
self.assert_not_in('file1', output)
# change the file
self.create_regular_file('file1', size=1024 * 100)
# should list the file as changed
output = self.cmd('create', '-v', '--filter=AM', self.repository_location + '::test3', 'input')
self.assert_in('file1', output)
def test_cmdline_compatibility(self): def test_cmdline_compatibility(self):
self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('file1', size=1024 * 80)
self.cmd('init', self.repository_location) self.cmd('init', self.repository_location)
@ -926,13 +950,13 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
return archive, repository return archive, repository
def test_check_usage(self): def test_check_usage(self):
output = self.cmd('check', self.repository_location, exit_code=0) output = self.cmd('check', '-v', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output) self.assert_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output) self.assert_in('Starting archive consistency check', output)
output = self.cmd('check', '--repository-only', self.repository_location, exit_code=0) output = self.cmd('check', '-v', '--repository-only', self.repository_location, exit_code=0)
self.assert_in('Starting repository check', output) self.assert_in('Starting repository check', output)
self.assert_not_in('Starting archive consistency check', output) self.assert_not_in('Starting archive consistency check', output)
output = self.cmd('check', '--archives-only', self.repository_location, exit_code=0) output = self.cmd('check', '-v', '--archives-only', self.repository_location, exit_code=0)
self.assert_not_in('Starting repository check', output) self.assert_not_in('Starting repository check', output)
self.assert_in('Starting archive consistency check', output) self.assert_in('Starting archive consistency check', output)
@ -968,7 +992,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
repository.delete(Manifest.MANIFEST_ID) repository.delete(Manifest.MANIFEST_ID)
repository.commit() repository.commit()
self.cmd('check', self.repository_location, exit_code=1) self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '--repair', self.repository_location, exit_code=0) output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive1', output) self.assert_in('archive1', output)
self.assert_in('archive2', output) self.assert_in('archive2', output)
self.cmd('check', self.repository_location, exit_code=0) self.cmd('check', self.repository_location, exit_code=0)

View file

@ -25,7 +25,9 @@ New features:
- implement borg break-lock command, fixes #157 - implement borg break-lock command, fixes #157
- include system info below traceback, fixes #324 - include system info below traceback, fixes #324
- use ISO-8601 date and time format, fixes #375 - use ISO-8601 date and time format, fixes #375
- add --log-level to set the level of the builtin logging configuration, fixes #426 - add --debug and --info (same as --verbose) to set the log level of the
builtin logging configuration (which otherwise defaults to warning),
fixes #426
- configure logging via env var BORG_LOGGING_CONF - configure logging via env var BORG_LOGGING_CONF
- add a --no-progress flag to forcibly disable progress info - add a --no-progress flag to forcibly disable progress info

View file

@ -19,6 +19,18 @@ separate sections either. The `flake8
<https://flake8.readthedocs.org/>`_ commandline tool should be used to <https://flake8.readthedocs.org/>`_ commandline tool should be used to
check for style errors before sending pull requests. check for style errors before sending pull requests.
Output and Logging
------------------
When writing logger calls, always use correct log level (debug only for
debugging, info for informative messages, warning for warnings, error for
errors, critical for critical errors/states).
When directly talking to the user (e.g. Y/N questions), do not use logging,
but directly output to stderr (not: stdout, it could be connected to a pipe).
To control the amount and kinds of messages output to stderr or emitted at
info level, use flags like --stats.
Building a development environment Building a development environment
---------------------------------- ----------------------------------

View file

@ -14,18 +14,20 @@ General
Type of log output Type of log output
~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~
You can set the log level of the builtin logging configuration using the The log level of the builtin logging configuration defaults to WARNING.
--log-level option. This is because we want |project_name| to be mostly silent and only output
warnings (plus errors and critical messages).
Supported levels: ``debug``, ``info``, ``warning``, ``error``, ``critical``. Use --verbose or --info to set INFO (you will get informative output then
additionally to warnings, errors, critical messages).
Use --debug to set DEBUG to get output made for debugging.
All log messages created with at least the given level will be output. All log messages created with at least the set level will be output.
Amount of informational output Log levels: DEBUG < INFO < WARNING < ERROR < CRITICAL
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
For some subcommands, using the ``-v`` or ``--verbose`` option will give you While you can set misc. log levels, do not expect that every command will
more informational output (at ``info`` level). give different output on different log levels - it's just a possibility.
Return codes Return codes
~~~~~~~~~~~~ ~~~~~~~~~~~~
@ -436,8 +438,13 @@ Item flags
~~~~~~~~~~ ~~~~~~~~~~
`borg create -v` outputs a verbose list of all files, directories and other `borg create -v` outputs a verbose list of all files, directories and other
file system items it considered. For each item, it prefixes a single-letter file system items it considered (no matter whether they had content changes
flag that indicates type and/or status of the item. or not). For each item, it prefixes a single-letter flag that indicates type
and/or status of the item.
If you are interested only in a subset of that output, you can give e.g.
`--filter=AME` and it will only show regular files with A, M or E status (see
below).
A uppercase character represents the status of a regular file relative to the A uppercase character represents the status of a regular file relative to the
"files" cache (not relative to the repo - this is an issue if the files cache "files" cache (not relative to the repo - this is an issue if the files cache