From 7b9d0c9739c459066caad9122eb92505d13a6c16 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 22 Sep 2016 02:43:57 +0200 Subject: [PATCH 01/11] yes(): abort on wrong answers, saying so except for the passphrase display as we can only display it as long as we have it in memory, here: retry, telling the user if he entered something invalid and needs to enter again. --- borg/archiver.py | 7 ++++--- borg/cache.py | 6 ++++-- borg/helpers.py | 5 ++--- borg/key.py | 5 +++-- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index bb8e33f77..f367a4a06 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -141,7 +141,8 @@ def do_check(self, args, repository): msg = ("'check --repair' is an experimental feature that might result in data loss." + "\n" + "Type 'YES' if you understand this and want to continue: ") - if not yes(msg, false_msg="Aborting.", truish=('YES', ), + if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", + truish=('YES', ), retry=False, env_var_override='BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'): return EXIT_ERROR if not args.archives_only: @@ -466,8 +467,8 @@ def do_delete(self, args, repository): msg.append(format_archive(archive_info)) msg.append("Type 'YES' if you understand this and want to continue: ") msg = '\n'.join(msg) - if not yes(msg, false_msg="Aborting.", truish=('YES', ), - env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): + if not yes(msg, false_msg="Aborting.", invalid_msg='Invalid answer, aborting.', truish=('YES', ), + retry=False, env_var_override='BORG_DELETE_I_KNOW_WHAT_I_AM_DOING'): self.exit_code = EXIT_ERROR return self.exit_code repository.destroy() diff --git a/borg/cache.py b/borg/cache.py index 763a262aa..b843fc49e 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -63,7 +63,8 @@ def __init__(self, repository, key, manifest, path=None, sync=True, do_files=Fal msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): + if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", + retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): raise self.CacheInitAbortedError() self.create() self.open(lock_wait=lock_wait) @@ -73,7 +74,8 @@ def __init__(self, repository, key, manifest, path=None, sync=True, do_files=Fal msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + "\n" + "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): + if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", + retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): raise self.RepositoryAccessAborted() if sync and self.manifest.id != self.manifest_id: diff --git a/borg/helpers.py b/borg/helpers.py index 27b3f0d3a..2867d4c97 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -959,9 +959,8 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, default=False, retry=True, env_var_override=None, ofile=None, input=input): """Output (usually a question) and let user input an answer. Qualifies the answer according to falsish, truish and defaultish as True, False or . - If it didn't qualify and retry_msg is None (no retries wanted), - return the default [which defaults to False]. Otherwise let user retry - answering until answer is qualified. + If it didn't qualify and retry is False (no retries wanted), return the default [which + defaults to False]. If retry is True let user retry answering until answer is qualified. If env_var_override is given and this var is present in the environment, do not ask the user, but just use the env var contents as answer as if it was typed in. diff --git a/borg/key.py b/borg/key.py index 95178f7c8..e88baf57f 100644 --- a/borg/key.py +++ b/borg/key.py @@ -190,8 +190,9 @@ def getpass(cls, prompt): @classmethod def verification(cls, passphrase): - if yes('Do you want your passphrase to be displayed for verification? [yN]: ', - env_var_override='BORG_DISPLAY_PASSPHRASE'): + msg = 'Do you want your passphrase to be displayed for verification? [yN]: ' + if yes(msg, retry_msg=msg, invalid_msg='Invalid answer, try again.', + retry=True, env_var_override='BORG_DISPLAY_PASSPHRASE'): print('Your passphrase (between double-quotes): "%s"' % passphrase, file=sys.stderr) print('Make sure the passphrase displayed above is exactly what you wanted.', From 29b5136da754b4f5959bc8771f44895b5195a605 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 15:05:39 +0200 Subject: [PATCH 02/11] archiver: Move key management commands to new key subcommand. --- borg/archiver.py | 12 ++++++++++-- borg/keymanager.py | 2 +- borg/testsuite/archiver.py | 18 +++++++++--------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6ef1d0fb8..8a07c5c6e 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1110,7 +1110,15 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) - subparser = subparsers.add_parser('key-export', parents=[common_parser], + subparser = subparsers.add_parser('key', + description="Manage a keyfile or repokey of a repository", + epilog="", + formatter_class=argparse.RawDescriptionHelpFormatter, + help='manage repository key') + + key_parsers = subparser.add_subparsers(title='required arguments', metavar='') + + subparser = key_parsers.add_parser('export', parents=[common_parser], description=self.do_key_export.__doc__, epilog="", formatter_class=argparse.RawDescriptionHelpFormatter, @@ -1124,7 +1132,7 @@ def build_parser(self, args=None, prog=None): default=False, help='Create an export suitable for printing and later type-in') - subparser = subparsers.add_parser('key-import', parents=[common_parser], + subparser = key_parsers.add_parser('import', parents=[common_parser], description=self.do_key_import.__doc__, epilog="", formatter_class=argparse.RawDescriptionHelpFormatter, diff --git a/borg/keymanager.py b/borg/keymanager.py index 244e16c69..8eef581da 100644 --- a/borg/keymanager.py +++ b/borg/keymanager.py @@ -98,7 +98,7 @@ def grouped(s): i += 1 return ret - export = 'To restore key use borg key-import --paper /path/to/repo\n\n' + export = 'To restore key use borg key import --paper /path/to/repo\n\n' binary = a2b_base64(self.keyblob) export += 'BORG PAPER KEY v1\n' diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d5896c038..8fb9eddf3 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1200,7 +1200,7 @@ def test_key_export_keyfile(self): export_file = self.output_path + '/exported' self.cmd('init', self.repository_location, '--encryption', 'keyfile') repo_id = self._extract_repository_id(self.repository_path) - self.cmd('key-export', self.repository_location, export_file) + self.cmd('key', 'export', self.repository_location, export_file) with open(export_file, 'r') as fd: export_contents = fd.read() @@ -1216,7 +1216,7 @@ def test_key_export_keyfile(self): os.unlink(key_file) - self.cmd('key-import', self.repository_location, export_file) + self.cmd('key', 'import', self.repository_location, export_file) with open(key_file, 'r') as fd: key_contents2 = fd.read() @@ -1227,7 +1227,7 @@ def test_key_export_repokey(self): export_file = self.output_path + '/exported' self.cmd('init', self.repository_location, '--encryption', 'repokey') repo_id = self._extract_repository_id(self.repository_path) - self.cmd('key-export', self.repository_location, export_file) + self.cmd('key', 'export', self.repository_location, export_file) with open(export_file, 'r') as fd: export_contents = fd.read() @@ -1246,7 +1246,7 @@ def test_key_export_repokey(self): with Repository(self.repository_path) as repository: repository.save_key(b'') - self.cmd('key-import', self.repository_location, export_file) + self.cmd('key', 'import', self.repository_location, export_file) with Repository(self.repository_path) as repository: repo_key2 = RepoKey(repository) @@ -1258,17 +1258,17 @@ def test_key_import_errors(self): export_file = self.output_path + '/exported' self.cmd('init', self.repository_location, '--encryption', 'keyfile') - self.cmd('key-import', self.repository_location, export_file, exit_code=EXIT_ERROR) + self.cmd('key', 'import', self.repository_location, export_file, exit_code=EXIT_ERROR) with open(export_file, 'w') as fd: fd.write('something not a key\n') - self.assert_raises(NotABorgKeyFile, lambda: self.cmd('key-import', self.repository_location, export_file)) + self.assert_raises(NotABorgKeyFile, lambda: self.cmd('key', 'import', self.repository_location, export_file)) with open(export_file, 'w') as fd: fd.write('BORG_KEY a0a0a0\n') - self.assert_raises(RepoIdMismatch, lambda: self.cmd('key-import', self.repository_location, export_file)) + self.assert_raises(RepoIdMismatch, lambda: self.cmd('key', 'import', self.repository_location, export_file)) def test_key_export_paperkey(self): repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' @@ -1283,12 +1283,12 @@ def test_key_export_paperkey(self): fd.write(KeyfileKey.FILE_ID + ' ' + repo_id + '\n') fd.write(b2a_base64(b'abcdefghijklmnopqrstu').decode()) - self.cmd('key-export', '--paper', self.repository_location, export_file) + self.cmd('key', 'export', '--paper', self.repository_location, export_file) with open(export_file, 'r') as fd: export_contents = fd.read() - assert export_contents == """To restore key use borg key-import --paper /path/to/repo + assert export_contents == """To restore key use borg key import --paper /path/to/repo BORG PAPER KEY v1 id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 From 5c2424831e8a68783e6912ceb854ccb474a7689a Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 15:25:02 +0200 Subject: [PATCH 03/11] archiver: Create a subcommmand debug for all debug-* commands The debug commands all should subcommands of a common debug command. This commit adds this command but keeps the old command names for 1.0.x. The plan is to remove them in 1.1.0. --- borg/archiver.py | 82 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index 8a07c5c6e..024910608 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1585,6 +1585,22 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('topic', metavar='TOPIC', type=str, nargs='?', help='additional help on TOPIC') + debug_epilog = textwrap.dedent(""" + These commands are not intended for normal use and potentially very + dangerous if used incorrectly. + + They exist to improve debugging capabilities without direct system access, e.g. + in case you ever run into some severe malfunction. Use them only if you know + what you are doing or if a trusted developer tells you what to do.""") + + subparser = subparsers.add_parser('debug', + description='debugging command (not intended for normal use)', + epilog=debug_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='debugging command (not intended for normal use)') + + debug_parsers = subparser.add_subparsers(title='required arguments', metavar='') + debug_info_epilog = textwrap.dedent(""" This command displays some system information that might be useful for bug reports and debugging problems. If a traceback happens, this information is @@ -1597,6 +1613,13 @@ def build_parser(self, args=None, prog=None): help='show system infos for debugging / bug reports (debug)') subparser.set_defaults(func=self.do_debug_info) + subparser = debug_parsers.add_parser('info', parents=[common_parser], + description=self.do_debug_info.__doc__, + epilog=debug_info_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='show system infos for debugging / bug reports (debug)') + subparser.set_defaults(func=self.do_debug_info) + debug_dump_archive_items_epilog = textwrap.dedent(""" This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. """) @@ -1610,6 +1633,16 @@ def build_parser(self, args=None, prog=None): type=location_validator(archive=True), help='archive to dump') + subparser = debug_parsers.add_parser('dump-archive-items', parents=[common_parser], + description=self.do_debug_dump_archive_items.__doc__, + epilog=debug_dump_archive_items_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='dump archive items (metadata) (debug)') + subparser.set_defaults(func=self.do_debug_dump_archive_items) + subparser.add_argument('location', metavar='ARCHIVE', + type=location_validator(archive=True), + help='archive to dump') + debug_dump_repo_objs_epilog = textwrap.dedent(""" This command dumps raw (but decrypted and decompressed) repo objects to files. """) @@ -1623,6 +1656,16 @@ def build_parser(self, args=None, prog=None): type=location_validator(archive=False), help='repo to dump') + subparser = debug_parsers.add_parser('dump-repo-objs', parents=[common_parser], + description=self.do_debug_dump_repo_objs.__doc__, + epilog=debug_dump_repo_objs_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='dump repo objects (debug)') + subparser.set_defaults(func=self.do_debug_dump_repo_objs) + subparser.add_argument('location', metavar='REPOSITORY', + type=location_validator(archive=False), + help='repo to dump') + debug_get_obj_epilog = textwrap.dedent(""" This command gets an object from the repository. """) @@ -1640,6 +1683,20 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('path', metavar='PATH', type=str, help='file to write object data into') + subparser = debug_parsers.add_parser('get-obj', parents=[common_parser], + description=self.do_debug_get_obj.__doc__, + epilog=debug_get_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='get object from repository (debug)') + subparser.set_defaults(func=self.do_debug_get_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('id', metavar='ID', type=str, + help='hex object ID to get from the repo') + subparser.add_argument('path', metavar='PATH', type=str, + help='file to write object data into') + debug_put_obj_epilog = textwrap.dedent(""" This command puts objects into the repository. """) @@ -1655,6 +1712,18 @@ def build_parser(self, args=None, prog=None): subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, help='file(s) to read and create object(s) from') + subparser = debug_parsers.add_parser('put-obj', parents=[common_parser], + description=self.do_debug_put_obj.__doc__, + epilog=debug_put_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='put object to repository (debug)') + subparser.set_defaults(func=self.do_debug_put_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, + help='file(s) to read and create object(s) from') + debug_delete_obj_epilog = textwrap.dedent(""" This command deletes objects from the repository. """) @@ -1669,6 +1738,19 @@ def build_parser(self, args=None, prog=None): help='repository to use') subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, help='hex object ID(s) to delete from the repo') + + subparser = debug_parsers.add_parser('delete-obj', parents=[common_parser], + description=self.do_debug_delete_obj.__doc__, + epilog=debug_delete_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='delete object from repository (debug)') + subparser.set_defaults(func=self.do_debug_delete_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to delete from the repo') + return parser def get_args(self, argv, cmd): From a11436cfb6e67f114936b32279c27223a611eb91 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 16:08:22 +0200 Subject: [PATCH 04/11] setup.py: Add subcommand support to build_usage. --- setup.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index b0103e06a..e0a18a47a 100644 --- a/setup.py +++ b/setup.py @@ -150,19 +150,33 @@ def finalize_options(self): def run(self): print('generating usage docs') + if not os.path.exists('docs/usage'): + os.mkdir('docs/usage') # allows us to build docs without the C modules fully loaded during help generation from borg.archiver import Archiver parser = Archiver().build_parser(prog='borg') + + self.generate_level("", parser, Archiver) + + def generate_level(self, prefix, parser, Archiver): + is_subcommand = False choices = {} for action in parser._actions: - if action.choices is not None: - choices.update(action.choices) + if action.choices is not None and 'SubParsersAction' in str(action.__class__): + is_subcommand = True + for cmd, parser in action.choices.items(): + choices[prefix + cmd] = parser + if prefix and not choices: + return print('found commands: %s' % list(choices.keys())) - if not os.path.exists('docs/usage'): - os.mkdir('docs/usage') + for command, parser in choices.items(): print('generating help for %s' % command) - with open('docs/usage/%s.rst.inc' % command, 'w') as doc: + + if self.generate_level(command + " ", parser, Archiver): + return + + with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") if command == 'help': for topic in Archiver.helptext: @@ -173,14 +187,16 @@ def run(self): doc.write(Archiver.helptext[topic]) else: params = {"command": command, + "command_": command.replace(' ', '_'), "underline": '-' * len('borg ' + command)} - doc.write(".. _borg_{command}:\n\n".format(**params)) + doc.write(".. _borg_{command_}:\n\n".format(**params)) doc.write("borg {command}\n{underline}\n::\n\n".format(**params)) epilog = parser.epilog parser.epilog = None doc.write(re.sub("^", " ", parser.format_help(), flags=re.M)) doc.write("\nDescription\n~~~~~~~~~~~\n") doc.write(epilog) + return is_subcommand class build_api(Command): From d9b880fdf32a0ec2463db27e6b650e73d59df12b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 23 Sep 2016 23:43:16 +0200 Subject: [PATCH 05/11] fix signal handling, fixes #1620 use context manager for signal handler installation / restoration - this includes the special case of installing handler SIG_IGN to ignore a signal and restoring the original (non-ignoring) handler. use SIG_IGN to avoid a 2nd signal interrupts the handling of the 1st signal. --- borg/archiver.py | 180 +++++++++++++++++++++-------------------------- borg/helpers.py | 48 +++++++++++++ 2 files changed, 128 insertions(+), 100 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 6ef1d0fb8..73402e981 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -23,6 +23,7 @@ Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \ EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper +from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm from .logger import create_logger, setup_logging logger = create_logger() from .compress import Compressor @@ -1697,59 +1698,28 @@ def run(self, args): return args.func(args) -def sig_info_handler(signum, stack): # pragma: no cover +def sig_info_handler(sig_no, stack): # pragma: no cover """search the stack for infos about the currently processed file and print them""" - for frame in inspect.getouterframes(stack): - func, loc = frame[3], frame[0].f_locals - if func in ('process_file', '_process', ): # create op - path = loc['path'] - try: - pos = loc['fd'].tell() - total = loc['st'].st_size - except Exception: - pos, total = 0, 0 - logger.info("{0} {1}/{2}".format(path, format_file_size(pos), format_file_size(total))) - break - if func in ('extract_item', ): # extract op - path = loc['item'][b'path'] - try: - pos = loc['fd'].tell() - except Exception: - pos = 0 - logger.info("{0} {1}/???".format(path, format_file_size(pos))) - break - - -class SIGTERMReceived(BaseException): - pass - - -def sig_term_handler(signum, stack): - raise SIGTERMReceived - - -class SIGHUPReceived(BaseException): - pass - - -def sig_hup_handler(signum, stack): - raise SIGHUPReceived - - -def setup_signal_handlers(): # pragma: no cover - sigs = [] - if hasattr(signal, 'SIGUSR1'): - sigs.append(signal.SIGUSR1) # kill -USR1 pid - if hasattr(signal, 'SIGINFO'): - sigs.append(signal.SIGINFO) # kill -INFO pid (or ctrl-t) - for sig in sigs: - signal.signal(sig, sig_info_handler) - # If we received SIGTERM or SIGHUP, catch them and raise a proper exception - # that can be handled for an orderly exit. SIGHUP is important especially - # for systemd systems, where logind sends it when a session exits, in - # addition to any traditional use. - signal.signal(signal.SIGTERM, sig_term_handler) - signal.signal(signal.SIGHUP, sig_hup_handler) + with signal_handler(sig_no, signal.SIG_IGN): + for frame in inspect.getouterframes(stack): + func, loc = frame[3], frame[0].f_locals + if func in ('process_file', '_process', ): # create op + path = loc['path'] + try: + pos = loc['fd'].tell() + total = loc['st'].st_size + except Exception: + pos, total = 0, 0 + logger.info("{0} {1}/{2}".format(path, format_file_size(pos), format_file_size(total))) + break + if func in ('extract_item', ): # extract op + path = loc['item'][b'path'] + try: + pos = loc['fd'].tell() + except Exception: + pos = 0 + logger.info("{0} {1}/???".format(path, format_file_size(pos))) + break def main(): # pragma: no cover @@ -1757,54 +1727,64 @@ def main(): # pragma: no cover # issues when print()-ing unicode file names sys.stdout = ErrorIgnoringTextIOWrapper(sys.stdout.buffer, sys.stdout.encoding, 'replace', line_buffering=True) sys.stderr = ErrorIgnoringTextIOWrapper(sys.stderr.buffer, sys.stderr.encoding, 'replace', line_buffering=True) - setup_signal_handlers() - archiver = Archiver() - msg = None - try: - args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) - except Error as e: - msg = e.get_message() - if e.traceback: - msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) - # we might not have logging setup yet, so get out quickly - print(msg, file=sys.stderr) - sys.exit(e.exit_code) - try: - exit_code = archiver.run(args) - except Error as e: - msg = e.get_message() - if e.traceback: - msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) - exit_code = e.exit_code - except RemoteRepository.RPCError as e: - msg = '%s\n%s' % (str(e), sysinfo()) - exit_code = EXIT_ERROR - except Exception: - msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo()) - exit_code = EXIT_ERROR - except KeyboardInterrupt: - msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo()) - exit_code = EXIT_ERROR - except SIGTERMReceived: - msg = 'Received SIGTERM.' - exit_code = EXIT_ERROR - except SIGHUPReceived: - msg = 'Received SIGHUP.' - exit_code = EXIT_ERROR - if msg: - logger.error(msg) - if args.show_rc: - exit_msg = 'terminating with %s status, rc %d' - if exit_code == EXIT_SUCCESS: - logger.info(exit_msg % ('success', exit_code)) - elif exit_code == EXIT_WARNING: - logger.warning(exit_msg % ('warning', exit_code)) - elif exit_code == EXIT_ERROR: - logger.error(exit_msg % ('error', exit_code)) - else: - # if you see 666 in output, it usually means exit_code was None - logger.error(exit_msg % ('abnormal', exit_code or 666)) - sys.exit(exit_code) + # If we receive SIGINT (ctrl-c), SIGTERM (kill) or SIGHUP (kill -HUP), + # catch them and raise a proper exception that can be handled for an + # orderly exit. + # SIGHUP is important especially for systemd systems, where logind + # sends it when a session exits, in addition to any traditional use. + # Output some info if we receive SIGUSR1 or SIGINFO (ctrl-t). + with signal_handler('SIGINT', raising_signal_handler(KeyboardInterrupt)), \ + signal_handler('SIGHUP', raising_signal_handler(SigHup)), \ + signal_handler('SIGTERM', raising_signal_handler(SigTerm)), \ + signal_handler('SIGUSR1', sig_info_handler), \ + signal_handler('SIGINFO', sig_info_handler): + archiver = Archiver() + msg = None + try: + args = archiver.get_args(sys.argv, os.environ.get('SSH_ORIGINAL_COMMAND')) + except Error as e: + msg = e.get_message() + if e.traceback: + msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) + # we might not have logging setup yet, so get out quickly + print(msg, file=sys.stderr) + sys.exit(e.exit_code) + try: + exit_code = archiver.run(args) + except Error as e: + msg = e.get_message() + if e.traceback: + msg += "\n%s\n%s" % (traceback.format_exc(), sysinfo()) + exit_code = e.exit_code + except RemoteRepository.RPCError as e: + msg = '%s\n%s' % (str(e), sysinfo()) + exit_code = EXIT_ERROR + except Exception: + msg = 'Local Exception.\n%s\n%s' % (traceback.format_exc(), sysinfo()) + exit_code = EXIT_ERROR + except KeyboardInterrupt: + msg = 'Keyboard interrupt.\n%s\n%s' % (traceback.format_exc(), sysinfo()) + exit_code = EXIT_ERROR + except SigTerm: + msg = 'Received SIGTERM.' + exit_code = EXIT_ERROR + except SigHup: + msg = 'Received SIGHUP.' + exit_code = EXIT_ERROR + if msg: + logger.error(msg) + if args.show_rc: + exit_msg = 'terminating with %s status, rc %d' + if exit_code == EXIT_SUCCESS: + logger.info(exit_msg % ('success', exit_code)) + elif exit_code == EXIT_WARNING: + logger.warning(exit_msg % ('warning', exit_code)) + elif exit_code == EXIT_ERROR: + logger.error(exit_msg % ('error', exit_code)) + else: + # if you see 666 in output, it usually means exit_code was None + logger.error(exit_msg % ('abnormal', exit_code or 666)) + sys.exit(exit_code) if __name__ == '__main__': diff --git a/borg/helpers.py b/borg/helpers.py index 24499c778..27555a4e8 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -1,5 +1,6 @@ import argparse from collections import namedtuple +import contextlib from functools import wraps import grp import os @@ -10,6 +11,7 @@ from shutil import get_terminal_size import sys import platform +import signal import threading import time import unicodedata @@ -1160,3 +1162,49 @@ def write(self, s): except OSError: pass return len(s) + + +class SignalException(BaseException): + """base class for all signal-based exceptions""" + + +class SigHup(SignalException): + """raised on SIGHUP signal""" + + +class SigTerm(SignalException): + """raised on SIGTERM signal""" + + +@contextlib.contextmanager +def signal_handler(sig, handler): + """ + when entering context, set up signal handler for signal . + when leaving context, restore original signal handler. + + can bei either a str when giving a signal.SIGXXX attribute name (it + won't crash if the attribute name does not exist as some names are platform + specific) or a int, when giving a signal number. + + is any handler value as accepted by the signal.signal(sig, handler). + """ + if isinstance(sig, str): + sig = getattr(signal, sig, None) + if sig is not None: + orig_handler = signal.signal(sig, handler) + try: + yield + finally: + if sig is not None: + signal.signal(sig, orig_handler) + + +def raising_signal_handler(exc_cls): + def handler(sig_no, frame): + # setting SIG_IGN avoids that an incoming second signal of this + # kind would raise a 2nd exception while we still process the + # exception handler for exc_cls for the 1st signal. + signal.signal(sig_no, signal.SIG_IGN) + raise exc_cls + + return handler From 4f9f25db026be595ea43d3dceef7397c908fd1ab Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 16:33:30 +0200 Subject: [PATCH 06/11] development.rst: Add sphinx_rtd_theme to the sphinx install command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s used by default, so install it as well. --- docs/development.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/development.rst b/docs/development.rst index 3e89e34c7..513bb4ec6 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -147,7 +147,7 @@ The documentation (in reStructuredText format, .rst) is in docs/. To build the html version of it, you need to have sphinx installed:: - pip3 install sphinx # important: this will install sphinx with Python 3 + pip3 install sphinx sphinx_rtd_theme # important: this will install sphinx with Python 3 Now run:: From 60c5482e6ba12ac68892b89ae2ccb23db7cae534 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Sep 2016 04:34:25 +0200 Subject: [PATCH 07/11] fix closed FD issue, fixes #1551 --- borg/repository.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/borg/repository.py b/borg/repository.py index 71e9040a6..fae22119a 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -705,6 +705,13 @@ def iter_objects(self, segment, include_data=False): else: yield tag, key, offset offset += size + # we must get the fd via get_fd() here again as we yielded to our caller and it might + # have triggered closing of the fd we had before (e.g. by calling io.read() for + # different segment(s)). + # by calling get_fd() here again we also make our fd "recently used" so it likely + # does not get kicked out of self.fds LRUcache. + fd = self.get_fd(segment) + fd.seek(offset) header = fd.read(self.header_fmt.size) def recover_segment(self, segment, filename): From 7c2025a2c0add837b7947c422de5003d5b696d81 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Mon, 26 Sep 2016 19:28:00 +0200 Subject: [PATCH 08/11] testsuite/archiver.py: Add a comment how to easily test ArchiverTestCaseBinary locally. --- borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8fb9eddf3..6c74cca04 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -69,7 +69,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw): sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr -# check if the binary "borg.exe" is available +# check if the binary "borg.exe" is available (for local testing a symlink to virtualenv/bin/borg should do) try: exec_cmd('help', exe='borg.exe', fork=True) BORG_EXES = ['python', 'binary', ] From 15444b19d1507fabf46291f7b216d883e3743f1f Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Mon, 26 Sep 2016 19:28:28 +0200 Subject: [PATCH 09/11] testsuite/archiver.py: Fix key import failure with ArchiverTestCaseBinary --- borg/testsuite/archiver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6c74cca04..e7f805ed4 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1263,12 +1263,18 @@ def test_key_import_errors(self): with open(export_file, 'w') as fd: fd.write('something not a key\n') - self.assert_raises(NotABorgKeyFile, lambda: self.cmd('key', 'import', self.repository_location, export_file)) + if self.FORK_DEFAULT: + self.cmd('key', 'import', self.repository_location, export_file, exit_code=2) + else: + self.assert_raises(NotABorgKeyFile, lambda: self.cmd('key', 'import', self.repository_location, export_file)) with open(export_file, 'w') as fd: fd.write('BORG_KEY a0a0a0\n') - self.assert_raises(RepoIdMismatch, lambda: self.cmd('key', 'import', self.repository_location, export_file)) + if self.FORK_DEFAULT: + self.cmd('key', 'import', self.repository_location, export_file, exit_code=2) + else: + self.assert_raises(RepoIdMismatch, lambda: self.cmd('key', 'import', self.repository_location, export_file)) def test_key_export_paperkey(self): repo_id = 'e294423506da4e1ea76e8dcdf1a3919624ae3ae496fddf905610c351d3f09239' From bb6c0cd2acc18c57735f81f27185eb37def90458 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 26 Sep 2016 20:08:04 +0200 Subject: [PATCH 10/11] vagrant: update FUSE for macOS --- Vagrantfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8316ec2f9..f00b8c14a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -61,9 +61,9 @@ def packages_darwin # install all the (security and other) updates sudo softwareupdate --install --all # get osxfuse 3.x pre-release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.4.1/osxfuse-3.4.1.dmg >osxfuse.dmg + curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.1/osxfuse-3.5.1.dmg >osxfuse.dmg MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.4.1.pkg" -target / + && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.1.pkg" -target / sudo chown -R vagrant /usr/local # brew must be able to create stuff here ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew update From b84014e7d9a7ba92224e3ffb1418e195cda0c9c5 Mon Sep 17 00:00:00 2001 From: textshell Date: Tue, 27 Sep 2016 15:24:31 +0200 Subject: [PATCH 11/11] archiver: Add documentation for "key export" and "key import" commands. (#1641) archiver: Add documentation for "key export" and "key import" commands --- borg/archiver.py | 31 +++++++++++++++++-- docs/quickstart.rst | 9 ++++-- docs/usage.rst | 6 ++++ docs/usage/key_export.rst.inc | 57 +++++++++++++++++++++++++++++++++++ docs/usage/key_import.rst.inc | 45 +++++++++++++++++++++++++++ 5 files changed, 144 insertions(+), 4 deletions(-) create mode 100644 docs/usage/key_export.rst.inc create mode 100644 docs/usage/key_import.rst.inc diff --git a/borg/archiver.py b/borg/archiver.py index 3c4682e5a..ce7655fc5 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1120,9 +1120,28 @@ def build_parser(self, args=None, prog=None): key_parsers = subparser.add_subparsers(title='required arguments', metavar='') + key_export_epilog = textwrap.dedent(""" + If repository encryption is used, the repository is inaccessible + without the key. This command allows to backup this essential key. + + There are two backup formats. The normal backup format is suitable for + digital storage as a file. The ``--paper`` backup format is optimized + for printing and typing in while importing, with per line checks to + reduce problems with manual input. + + For repositories using keyfile encryption the key is saved locally + on the system that is capable of doing backups. To guard against loss + of this key, the key needs to be backed up independently of the main + data backup. + + For repositories using the repokey encryption the key is saved in the + repository in the config file. A backup is thus not strictly needed, + but guards against the repository becoming inaccessible if the file + is damaged for some reason. + """) subparser = key_parsers.add_parser('export', parents=[common_parser], description=self.do_key_export.__doc__, - epilog="", + epilog=key_export_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, help='export repository key for backup') subparser.set_defaults(func=self.do_key_export) @@ -1134,9 +1153,17 @@ def build_parser(self, args=None, prog=None): default=False, help='Create an export suitable for printing and later type-in') + key_import_epilog = textwrap.dedent(""" + This command allows to restore a key previously backed up with the + export command. + + If the ``--paper`` option is given, the import will be an interactive + process in which each line is checked for plausibility before + proceeding to the next line. For this format PATH must not be given. + """) subparser = key_parsers.add_parser('import', parents=[common_parser], description=self.do_key_import.__doc__, - epilog="", + epilog=key_import_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, help='import repository key from backup') subparser.set_defaults(func=self.do_key_import) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 20cd32d1d..f7105d6a7 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -228,8 +228,13 @@ For automated backups the passphrase can be specified using the the key in case it gets corrupted or lost. Also keep your passphrase at a safe place. - The backup that is encrypted with that key/passphrase won't help you - with that, of course. + You can make backups using :ref:`borg_key_export` subcommand. + + If you want to print a backup of your key to paper use the ``--paper`` + option of this command and print the result. + + A backup inside of the backup that is encrypted with that key/passphrase + won't help you with that, of course. .. _remote_repos: diff --git a/docs/usage.rst b/docs/usage.rst index 89a9e3cc7..a3015f61b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -484,6 +484,12 @@ Examples $ fusermount -u /tmp/mymountpoint +.. include:: usage/key_export.rst.inc + + +.. include:: usage/key_import.rst.inc + + .. include:: usage/change-passphrase.rst.inc Examples diff --git a/docs/usage/key_export.rst.inc b/docs/usage/key_export.rst.inc new file mode 100644 index 000000000..47e9e119e --- /dev/null +++ b/docs/usage/key_export.rst.inc @@ -0,0 +1,57 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_key_export: + +borg key export +--------------- +:: + + usage: borg key export [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + [--paper] + [REPOSITORY] [PATH] + + Export the repository key for backup + + positional arguments: + REPOSITORY + PATH where to store the backup + + optional arguments: + -h, --help show this help message and exit + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + --paper Create an export suitable for printing and later type- + in + +Description +~~~~~~~~~~~ + +If repository encryption is used, the repository is inaccessible +without the key. This command allows to backup this essential key. + +There are two backup formats. The normal backup format is suitable for +digital storage as a file. The ``--paper`` backup format is optimized for +print out and later type-in, with per line checks to reduce problems +with manual input. + +For repositories using keyfile encryption the key is saved locally +on the system that is capable of doing backups. To guard against loss +of this key the key needs to be backed up independent of the main data +backup. + +For repositories using the repokey encryption the key is saved in the +repository in the config file. A backup is thus not strictly needed, +but guards against the repository becoming inaccessible if the file +is damaged for some reason. diff --git a/docs/usage/key_import.rst.inc b/docs/usage/key_import.rst.inc new file mode 100644 index 000000000..71a8eed46 --- /dev/null +++ b/docs/usage/key_import.rst.inc @@ -0,0 +1,45 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_key_import: + +borg key import +--------------- +:: + + usage: borg key import [-h] [--critical] [--error] [--warning] [--info] + [--debug] [--lock-wait N] [--show-rc] + [--no-files-cache] [--umask M] [--remote-path PATH] + [--paper] + [REPOSITORY] [PATH] + + Import the repository key from backup + + positional arguments: + REPOSITORY + PATH path to the backup + + optional arguments: + -h, --help show this help message and exit + --critical work on log level CRITICAL + --error work on log level ERROR + --warning work on log level WARNING (default) + --info, -v, --verbose + work on log level INFO + --debug work on log level DEBUG + --lock-wait N wait for the lock, but max. N seconds (default: 1). + --show-rc show/log the return code (rc) + --no-files-cache do not load/update the file metadata cache used to + detect unchanged files + --umask M set umask to M (local and remote, default: 0077) + --remote-path PATH set remote path to executable (default: "borg") + --paper interactively import from a backup done with --paper + +Description +~~~~~~~~~~~ + +This command allows to restore a key previously backed up with the +export command. + +If the ``--paper`` option is given, the import will be an interactive +process in which each line is checked for plausibility before +proceeding to the next line. For this format PATH must not be given.