From 1c261f6b7b497abd6896ba0f9d7bfb08821ca2be Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Fri, 2 Dec 2016 13:25:24 +0100 Subject: [PATCH 1/9] setup.py: fix build_usage not processing all commands --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d51941568..e321d41fe 100644 --- a/setup.py +++ b/setup.py @@ -185,7 +185,7 @@ class build_usage(Command): print('generating help for %s' % command) if self.generate_level(command + " ", parser, Archiver): - break + continue with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") From 288cac788c677013068ea2025605214941b626c1 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Fri, 2 Dec 2016 13:28:49 +0100 Subject: [PATCH 2/9] setup.py: build_usage: don't generate includes for debug commands --- docs/usage/debug-delete-obj.rst.inc | 39 --------------------- docs/usage/debug-dump-archive-items.rst.inc | 38 -------------------- docs/usage/debug-dump-repo-objs.rst.inc | 38 -------------------- docs/usage/debug-get-obj.rst.inc | 39 --------------------- docs/usage/debug-info.rst.inc | 35 ------------------ docs/usage/debug-put-obj.rst.inc | 38 -------------------- setup.py | 5 ++- 7 files changed, 4 insertions(+), 228 deletions(-) delete mode 100644 docs/usage/debug-delete-obj.rst.inc delete mode 100644 docs/usage/debug-dump-archive-items.rst.inc delete mode 100644 docs/usage/debug-dump-repo-objs.rst.inc delete mode 100644 docs/usage/debug-get-obj.rst.inc delete mode 100644 docs/usage/debug-info.rst.inc delete mode 100644 docs/usage/debug-put-obj.rst.inc diff --git a/docs/usage/debug-delete-obj.rst.inc b/docs/usage/debug-delete-obj.rst.inc deleted file mode 100644 index 7ed144bfb..000000000 --- a/docs/usage/debug-delete-obj.rst.inc +++ /dev/null @@ -1,39 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-delete-obj: - -borg debug-delete-obj ---------------------- -:: - - usage: borg debug-delete-obj [-h] [--critical] [--error] [--warning] [--info] - [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] - [--remote-path PATH] - [REPOSITORY] IDs [IDs ...] - - delete the objects with the given IDs from the repo - - positional arguments: - REPOSITORY repository to use - IDs hex object ID(s) to delete from the repo - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command deletes objects from the repository. diff --git a/docs/usage/debug-dump-archive-items.rst.inc b/docs/usage/debug-dump-archive-items.rst.inc deleted file mode 100644 index e83bbf854..000000000 --- a/docs/usage/debug-dump-archive-items.rst.inc +++ /dev/null @@ -1,38 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-dump-archive-items: - -borg debug-dump-archive-items ------------------------------ -:: - - usage: borg debug-dump-archive-items [-h] [--critical] [--error] [--warning] - [--info] [--debug] [--lock-wait N] - [--show-rc] [--no-files-cache] - [--umask M] [--remote-path PATH] - ARCHIVE - - dump (decrypted, decompressed) archive items metadata (not: data) - - positional arguments: - ARCHIVE archive to dump - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. diff --git a/docs/usage/debug-dump-repo-objs.rst.inc b/docs/usage/debug-dump-repo-objs.rst.inc deleted file mode 100644 index 4fcd45ae8..000000000 --- a/docs/usage/debug-dump-repo-objs.rst.inc +++ /dev/null @@ -1,38 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-dump-repo-objs: - -borg debug-dump-repo-objs -------------------------- -:: - - usage: borg debug-dump-repo-objs [-h] [--critical] [--error] [--warning] - [--info] [--debug] [--lock-wait N] - [--show-rc] [--no-files-cache] [--umask M] - [--remote-path PATH] - REPOSITORY - - dump (decrypted, decompressed) repo objects - - positional arguments: - REPOSITORY repo to dump - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command dumps raw (but decrypted and decompressed) repo objects to files. diff --git a/docs/usage/debug-get-obj.rst.inc b/docs/usage/debug-get-obj.rst.inc deleted file mode 100644 index f38f0e884..000000000 --- a/docs/usage/debug-get-obj.rst.inc +++ /dev/null @@ -1,39 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-get-obj: - -borg debug-get-obj ------------------- -:: - - usage: borg debug-get-obj [-h] [--critical] [--error] [--warning] [--info] - [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [REPOSITORY] ID PATH - - get object contents from the repository and write it into file - - positional arguments: - REPOSITORY repository to use - ID hex object ID to get from the repo - PATH file to write object data into - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command gets an object from the repository. diff --git a/docs/usage/debug-info.rst.inc b/docs/usage/debug-info.rst.inc deleted file mode 100644 index 2f4f7237f..000000000 --- a/docs/usage/debug-info.rst.inc +++ /dev/null @@ -1,35 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-info: - -borg debug-info ---------------- -:: - - usage: borg debug-info [-h] [--critical] [--error] [--warning] [--info] - [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - - display system information for debugging / bug reports - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command displays some system information that might be useful for bug -reports and debugging problems. If a traceback happens, this information is -already appended at the end of the traceback. diff --git a/docs/usage/debug-put-obj.rst.inc b/docs/usage/debug-put-obj.rst.inc deleted file mode 100644 index 4f02324bf..000000000 --- a/docs/usage/debug-put-obj.rst.inc +++ /dev/null @@ -1,38 +0,0 @@ -.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! - -.. _borg_debug-put-obj: - -borg debug-put-obj ------------------- -:: - - usage: borg debug-put-obj [-h] [--critical] [--error] [--warning] [--info] - [--debug] [--lock-wait N] [--show-rc] - [--no-files-cache] [--umask M] [--remote-path PATH] - [REPOSITORY] PATH [PATH ...] - - put file(s) contents into the repository - - positional arguments: - REPOSITORY repository to use - PATH file(s) to read and create object(s) from - - optional arguments: - -h, --help show this help message and exit - --critical work on log level CRITICAL - --error work on log level ERROR - --warning work on log level WARNING (default) - --info, -v, --verbose - work on log level INFO - --debug work on log level DEBUG - --lock-wait N wait for the lock, but max. N seconds (default: 1). - --show-rc show/log the return code (rc) - --no-files-cache do not load/update the file metadata cache used to - detect unchanged files - --umask M set umask to M (local and remote, default: 0077) - --remote-path PATH set remote path to executable (default: "borg") - -Description -~~~~~~~~~~~ - -This command puts objects into the repository. diff --git a/setup.py b/setup.py index e321d41fe..00e48f3f3 100644 --- a/setup.py +++ b/setup.py @@ -181,7 +181,10 @@ class build_usage(Command): return print('found commands: %s' % list(choices.keys())) - for command, parser in choices.items(): + for command, parser in sorted(choices.items()): + if command.startswith('debug'): + print('skipping', command) + continue print('generating help for %s' % command) if self.generate_level(command + " ", parser, Archiver): From dfd37d09d5981dd9f58ebac60175b8b2c2897457 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 3 Dec 2016 18:34:34 +0100 Subject: [PATCH 3/9] add a PR template pointing to guidelines --- .github/PULL_REQUEST_TEMPLATE | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 000000000..f01ff53c6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,8 @@ +Thank you for contributing code to Borg, your help is appreciated! + +Please, before you submit a pull request, make sure it complies with the +guidelines given in our documentation: + +https://borgbackup.readthedocs.io/en/latest/development.html#contributions + +**Please remove all above text before submitting your pull request.** From 335d599db4f812b3ad6a87bb3a2b31a6c9a772a9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 9 Dec 2016 03:37:13 +0100 Subject: [PATCH 4/9] fix location parser for archives with @ char, add test, fixes #1930 we must exclude colon and slash chars from the username, otherwise the term for the user part will match everything up to a @ char in the archive name. a slash can't be in a username as the home directory would contain a illegal slash (slash is path sep), a colon likely also should not be in a username because chown user:group ... syntax. --- borg/helpers.py | 4 ++-- borg/testsuite/helpers.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 72129f08e..2eae2ac1e 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -791,7 +791,7 @@ class Location: # regexes for misc. kinds of supported location specifiers: ssh_re = re.compile(r""" (?Pssh):// # ssh:// - (?:(?P[^@]+)@)? # user@ (optional) + (?:(?P[^@:/]+)@)? # user@ (optional) - username must not contain : or / (?P[^:/]+)(?::(?P\d+))? # host or host:port """ + path_re + optional_archive_re, re.VERBOSE) # path or path::archive @@ -802,7 +802,7 @@ class Location: # note: scp_re is also use for local pathes scp_re = re.compile(r""" ( - (?:(?P[^@]+)@)? # user@ (optional) + (?:(?P[^@:/]+)@)? # user@ (optional) - username must not contain : or / (?P[^:/]+): # host: (don't match / in host to disambiguate from file:) )? # user@host: part is optional """ + path_re + optional_archive_re, re.VERBOSE) # path with optional archive diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index d0fa86b6b..c6dadbece 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -94,6 +94,13 @@ class TestLocationWithoutEnv: assert repr(Location('/abs/path:with:colons')) == \ "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)" + def test_user_parsing(self): + # see issue #1930 + assert repr(Location('host:path::2016-12-31@23:59:59')) == \ + "Location(proto='ssh', user=None, host='host', port=None, path='path', archive='2016-12-31@23:59:59')" + assert repr(Location('ssh://host/path::2016-12-31@23:59:59')) == \ + "Location(proto='ssh', user=None, host='host', port=None, path='/path', archive='2016-12-31@23:59:59')" + def test_underspecified(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) with pytest.raises(ValueError): From bd8b4a44897e64b52b51d4ae492274b9ad3adf52 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 9 Dec 2016 04:42:23 +0100 Subject: [PATCH 5/9] update CHANGES (1.0-maint) --- docs/changes.rst | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 4b13273fa..7a79c61c0 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -71,6 +71,32 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.0.9 (not released yet) +-------------------------------- + +Bug fixes: + +- borg check: + + - rebuild manifest if it's corrupted + - skip corrupted chunks during manifest rebuild +- fix TypeError in errorhandler, #1903, #1894 + +Other changes: + +- docs: + + - add python3-devel as a dependency for cygwin-based installation + - clarify extract is relative to current directory + - FAQ: fix link to changelog + - markup fixes +- tests: test_get_(cache|keys)_dir: clean env state, #1897 +- setup.py build_usage: + + - fixed build_usage not processing all commands + - fixed build_usage not generating includes for debug commands + + Version 1.0.9rc1 (2016-11-27) ----------------------------- From c6017abfb7bd32f3cd598ad8df6123bd02d803b3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 13 Dec 2016 19:27:01 +0100 Subject: [PATCH 6/9] get back pytest's pretty assertion failures, fixes #1938 --- conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conftest.py b/conftest.py index d5423660b..d622e6df1 100644 --- a/conftest.py +++ b/conftest.py @@ -4,6 +4,8 @@ import sys import pytest +# needed to get pretty assertion failures in unit tests: +pytest.register_assert_rewrite('borg') # This is a hack to fix path problems because "borg" (the package) is in the source root. # When importing the conftest an "import borg" can incorrectly import the borg from the From 292ff42655ff4db95b3405952491cd42399ed3ea Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 9 Dec 2016 03:54:20 +0100 Subject: [PATCH 7/9] refactor common regex part into optional_user_re --- borg/helpers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 2eae2ac1e..ff7d16075 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -773,6 +773,17 @@ class Location: """ proto = user = host = port = path = archive = None + # user must not contain "@", ":" or "/". + # Quoting adduser error message: + # "To avoid problems, the username should consist only of letters, digits, + # underscores, periods, at signs and dashes, and not start with a dash + # (as defined by IEEE Std 1003.1-2001)." + # We use "@" as separator between username and hostname, so we must + # disallow it within the pure username part. + optional_user_re = r""" + (?:(?P[^@:/]+)@)? + """ + # path must not contain :: (it ends at :: or string end), but may contain single colons. # to avoid ambiguities with other regexes, it must also not start with ":". path_re = r""" @@ -791,7 +802,7 @@ class Location: # regexes for misc. kinds of supported location specifiers: ssh_re = re.compile(r""" (?Pssh):// # ssh:// - (?:(?P[^@:/]+)@)? # user@ (optional) - username must not contain : or / + """ + optional_user_re + r""" # user@ (optional) (?P[^:/]+)(?::(?P\d+))? # host or host:port """ + path_re + optional_archive_re, re.VERBOSE) # path or path::archive @@ -802,7 +813,7 @@ class Location: # note: scp_re is also use for local pathes scp_re = re.compile(r""" ( - (?:(?P[^@:/]+)@)? # user@ (optional) - username must not contain : or / + """ + optional_user_re + r""" # user@ (optional) (?P[^:/]+): # host: (don't match / in host to disambiguate from file:) )? # user@host: part is optional """ + path_re + optional_archive_re, re.VERBOSE) # path with optional archive From 60bbd7a944f101e3414b4c2feb941ba1ceb9bdb1 Mon Sep 17 00:00:00 2001 From: TW Date: Wed, 14 Dec 2016 01:29:43 +0100 Subject: [PATCH 8/9] update CHANGES (1.0-maint) (#1954) --- docs/changes.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 7a79c61c0..4ff900e7d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -80,7 +80,8 @@ Bug fixes: - rebuild manifest if it's corrupted - skip corrupted chunks during manifest rebuild -- fix TypeError in errorhandler, #1903, #1894 +- fix TypeError in integrity error handler, #1903, #1894 +- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 Other changes: @@ -90,7 +91,10 @@ Other changes: - clarify extract is relative to current directory - FAQ: fix link to changelog - markup fixes -- tests: test_get_(cache|keys)_dir: clean env state, #1897 +- tests: + + - test_get_(cache|keys)_dir: clean env state, #1897 + - get back pytest's pretty assertion failures, #1938 - setup.py build_usage: - fixed build_usage not processing all commands From 5a40870416a85ab08c3dca0d9b62808450cf2101 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 14 Dec 2016 00:00:09 +0100 Subject: [PATCH 9/9] add a borg debug/key dummy command, fixes #1932 the problem was that there neither was a do_debug implementation for the case someone just enters "borg debug", nor did the parser inherit from common_parser (so accessing .umask triggered an exception before setup_logging() was called, which triggered another exception when log output should have been emitted). same for do_key ("borg key"). added a generic handler that just prints the subcommand help. --- borg/archiver.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index bd3382251..cb02d4d8a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -992,6 +992,11 @@ class Archiver: parser.error('No help available on %s' % (args.topic,)) return self.exit_code + def do_subcommand_help(self, parser, args): + """display infos about subcommand""" + parser.print_help() + return EXIT_SUCCESS + def preprocess_args(self, args): deprecations = [ # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), @@ -1148,13 +1153,14 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) - subparser = subparsers.add_parser('key', + subparser = subparsers.add_parser('key', parents=[common_parser], description="Manage a keyfile or repokey of a repository", epilog="", formatter_class=argparse.RawDescriptionHelpFormatter, help='manage repository key') key_parsers = subparser.add_subparsers(title='required arguments', metavar='') + subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser)) key_export_epilog = textwrap.dedent(""" If repository encryption is used, the repository is inaccessible @@ -1681,13 +1687,14 @@ class Archiver: in case you ever run into some severe malfunction. Use them only if you know what you are doing or if a trusted developer tells you what to do.""") - subparser = subparsers.add_parser('debug', + subparser = subparsers.add_parser('debug', parents=[common_parser], description='debugging command (not intended for normal use)', epilog=debug_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, help='debugging command (not intended for normal use)') debug_parsers = subparser.add_subparsers(title='required arguments', metavar='') + subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser)) debug_info_epilog = textwrap.dedent(""" This command displays some system information that might be useful for bug