diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 000000000..f01ff53c6 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,8 @@ +Thank you for contributing code to Borg, your help is appreciated! + +Please, before you submit a pull request, make sure it complies with the +guidelines given in our documentation: + +https://borgbackup.readthedocs.io/en/latest/development.html#contributions + +**Please remove all above text before submitting your pull request.** diff --git a/conftest.py b/conftest.py index 312f57a20..d28672e47 100644 --- a/conftest.py +++ b/conftest.py @@ -2,6 +2,9 @@ import os import pytest +# needed to get pretty assertion failures in unit tests: +pytest.register_assert_rewrite('borg') + from borg.logger import setup_logging # Ensure that the loggers exist for all tests diff --git a/docs/changes.rst b/docs/changes.rst index 3308b7627..1678d0599 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -71,6 +71,36 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.0.9 (not released yet) +-------------------------------- + +Bug fixes: + +- borg check: + + - rebuild manifest if it's corrupted + - skip corrupted chunks during manifest rebuild +- fix TypeError in integrity error handler, #1903, #1894 +- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 + +Other changes: + +- docs: + + - add python3-devel as a dependency for cygwin-based installation + - clarify extract is relative to current directory + - FAQ: fix link to changelog + - markup fixes +- tests: + + - test_get_(cache|keys)_dir: clean env state, #1897 + - get back pytest's pretty assertion failures, #1938 +- setup.py build_usage: + + - fixed build_usage not processing all commands + - fixed build_usage not generating includes for debug commands + + Version 1.0.9rc1 (2016-11-27) ----------------------------- diff --git a/docs/usage/debug_delete-obj.rst.inc b/docs/usage/debug_delete-obj.rst.inc new file mode 100644 index 000000000..71248e1d7 --- /dev/null +++ b/docs/usage/debug_delete-obj.rst.inc @@ -0,0 +1,23 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_delete-obj: + +borg debug delete-obj +--------------------- +:: + + borg debug delete-obj REPOSITORY IDs + +positional arguments + REPOSITORY + repository to use + IDs + hex object ID(s) to delete from the repo + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command deletes objects from the repository. diff --git a/docs/usage/debug_dump-archive-items.rst.inc b/docs/usage/debug_dump-archive-items.rst.inc new file mode 100644 index 000000000..a28711536 --- /dev/null +++ b/docs/usage/debug_dump-archive-items.rst.inc @@ -0,0 +1,21 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_dump-archive-items: + +borg debug dump-archive-items +----------------------------- +:: + + borg debug dump-archive-items ARCHIVE + +positional arguments + ARCHIVE + archive to dump + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files. diff --git a/docs/usage/debug_dump-repo-objs.rst.inc b/docs/usage/debug_dump-repo-objs.rst.inc new file mode 100644 index 000000000..e041ecc18 --- /dev/null +++ b/docs/usage/debug_dump-repo-objs.rst.inc @@ -0,0 +1,21 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_dump-repo-objs: + +borg debug dump-repo-objs +------------------------- +:: + + borg debug dump-repo-objs REPOSITORY + +positional arguments + REPOSITORY + repo to dump + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command dumps raw (but decrypted and decompressed) repo objects to files. diff --git a/docs/usage/debug_get-obj.rst.inc b/docs/usage/debug_get-obj.rst.inc new file mode 100644 index 000000000..8a966314e --- /dev/null +++ b/docs/usage/debug_get-obj.rst.inc @@ -0,0 +1,25 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_get-obj: + +borg debug get-obj +------------------ +:: + + borg debug get-obj REPOSITORY ID PATH + +positional arguments + REPOSITORY + repository to use + ID + hex object ID to get from the repo + PATH + file to write object data into + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command gets an object from the repository. diff --git a/docs/usage/debug_info.rst.inc b/docs/usage/debug_info.rst.inc new file mode 100644 index 000000000..ccfbeb197 --- /dev/null +++ b/docs/usage/debug_info.rst.inc @@ -0,0 +1,19 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_info: + +borg debug info +--------------- +:: + + borg debug info + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command displays some system information that might be useful for bug +reports and debugging problems. If a traceback happens, this information is +already appended at the end of the traceback. diff --git a/docs/usage/debug_put-obj.rst.inc b/docs/usage/debug_put-obj.rst.inc new file mode 100644 index 000000000..5563593bd --- /dev/null +++ b/docs/usage/debug_put-obj.rst.inc @@ -0,0 +1,23 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_put-obj: + +borg debug put-obj +------------------ +:: + + borg debug put-obj REPOSITORY PATH + +positional arguments + REPOSITORY + repository to use + PATH + file(s) to read and create object(s) from + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command puts objects into the repository. diff --git a/docs/usage/debug_refcount-obj.rst.inc b/docs/usage/debug_refcount-obj.rst.inc new file mode 100644 index 000000000..c38fcc7a1 --- /dev/null +++ b/docs/usage/debug_refcount-obj.rst.inc @@ -0,0 +1,23 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_debug_refcount-obj: + +borg debug refcount-obj +----------------------- +:: + + borg debug refcount-obj REPOSITORY IDs + +positional arguments + REPOSITORY + repository to use + IDs + hex object ID(s) to show refcounts for + +`Common options`_ + | + +Description +~~~~~~~~~~~ + +This command displays the reference count for objects from the repository. diff --git a/setup.py b/setup.py index 5dfcbb306..6ef6d543b 100644 --- a/setup.py +++ b/setup.py @@ -226,11 +226,14 @@ class build_usage(Command): return print('found commands: %s' % list(choices.keys())) - for command, parser in choices.items(): + for command, parser in sorted(choices.items()): + if command.startswith('debug'): + print('skipping', command) + continue print('generating help for %s' % command) if self.generate_level(command + " ", parser, Archiver): - break + continue with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc: doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n") diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 7c51d034a..ee2e3e755 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1485,6 +1485,11 @@ class Archiver: parser.error('No help available on %s' % (args.topic,)) return self.exit_code + def do_subcommand_help(self, parser, args): + """display infos about subcommand""" + parser.print_help() + return EXIT_SUCCESS + def preprocess_args(self, args): deprecations = [ # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'), @@ -1723,13 +1728,14 @@ class Archiver: subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', type=location_validator(archive=False)) - subparser = subparsers.add_parser('key', add_help=True, + subparser = subparsers.add_parser('key', parents=[common_parser], add_help=False, description="Manage a keyfile or repokey of a repository", epilog="", formatter_class=argparse.RawDescriptionHelpFormatter, help='manage repository key') key_parsers = subparser.add_subparsers(title='required arguments', metavar='') + subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser)) key_export_epilog = textwrap.dedent(""" If repository encryption is used, the repository is inaccessible @@ -2512,13 +2518,14 @@ class Archiver: in case you ever run into some severe malfunction. Use them only if you know what you are doing or if a trusted developer tells you what to do.""") - subparser = subparsers.add_parser('debug', add_help=True, + subparser = subparsers.add_parser('debug', parents=[common_parser], add_help=False, description='debugging command (not intended for normal use)', epilog=debug_epilog, formatter_class=argparse.RawDescriptionHelpFormatter, help='debugging command (not intended for normal use)') debug_parsers = subparser.add_subparsers(title='required arguments', metavar='') + subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser)) debug_info_epilog = textwrap.dedent(""" This command displays some system information that might be useful for bug diff --git a/src/borg/helpers.py b/src/borg/helpers.py index cd6af5cd8..0c98f3a38 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -889,6 +889,17 @@ class Location: """ proto = user = host = port = path = archive = None + # user must not contain "@", ":" or "/". + # Quoting adduser error message: + # "To avoid problems, the username should consist only of letters, digits, + # underscores, periods, at signs and dashes, and not start with a dash + # (as defined by IEEE Std 1003.1-2001)." + # We use "@" as separator between username and hostname, so we must + # disallow it within the pure username part. + optional_user_re = r""" + (?:(?P[^@:/]+)@)? + """ + # path must not contain :: (it ends at :: or string end), but may contain single colons. # to avoid ambiguities with other regexes, it must also not start with ":". path_re = r""" @@ -907,7 +918,7 @@ class Location: # regexes for misc. kinds of supported location specifiers: ssh_re = re.compile(r""" (?Pssh):// # ssh:// - (?:(?P[^@]+)@)? # user@ (optional) + """ + optional_user_re + r""" # user@ (optional) (?P[^:/]+)(?::(?P\d+))? # host or host:port """ + path_re + optional_archive_re, re.VERBOSE) # path or path::archive @@ -918,7 +929,7 @@ class Location: # note: scp_re is also use for local pathes scp_re = re.compile(r""" ( - (?:(?P[^@]+)@)? # user@ (optional) + """ + optional_user_re + r""" # user@ (optional) (?P[^:/]+): # host: (don't match / in host to disambiguate from file:) )? # user@host: part is optional """ + path_re + optional_archive_re, re.VERBOSE) # path with optional archive diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 8c5306e87..1bc1ad7cc 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -97,6 +97,13 @@ class TestLocationWithoutEnv: assert repr(Location('/abs/path:with:colons')) == \ "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)" + def test_user_parsing(self): + # see issue #1930 + assert repr(Location('host:path::2016-12-31@23:59:59')) == \ + "Location(proto='ssh', user=None, host='host', port=None, path='path', archive='2016-12-31@23:59:59')" + assert repr(Location('ssh://host/path::2016-12-31@23:59:59')) == \ + "Location(proto='ssh', user=None, host='host', port=None, path='/path', archive='2016-12-31@23:59:59')" + def test_underspecified(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) with pytest.raises(ValueError):