Merge branch '1.0-maint'

.github/PULL_REQUEST_TEMPLATE

@@ -0,0 +1,8 @@
+Thank you for contributing code to Borg, your help is appreciated!
+
+Please, before you submit a pull request, make sure it complies with the
+guidelines given in our documentation:
+
+https://borgbackup.readthedocs.io/en/latest/development.html#contributions
+
+**Please remove all above text before submitting your pull request.**

conftest.py

@@ -2,6 +2,9 @@
 
 import pytest
 
+# needed to get pretty assertion failures in unit tests:
+pytest.register_assert_rewrite('borg')
+
 from borg.logger import setup_logging
 
 # Ensure that the loggers exist for all tests

docs/changes.rst

@@ -71,6 +71,36 @@ The best check that everything is ok is to run a dry-run extraction::
 Changelog
 =========
 
+Version 1.0.9 (not released yet)
+--------------------------------
+
+Bug fixes:
+
+- borg check:
+
+  - rebuild manifest if it's corrupted
+  - skip corrupted chunks during manifest rebuild
+- fix TypeError in integrity error handler, #1903, #1894
+- fix location parser for archives with @ char (regression introduced in 1.0.8), #1930
+
+Other changes:
+
+- docs:
+
+  - add python3-devel as a dependency for cygwin-based installation
+  - clarify extract is relative to current directory
+  - FAQ: fix link to changelog
+  - markup fixes
+- tests:
+
+  - test_get_(cache|keys)_dir: clean env state, #1897
+  - get back pytest's pretty assertion failures, #1938
+- setup.py build_usage:
+
+  - fixed build_usage not processing all commands
+  - fixed build_usage not generating includes for debug commands
+
+
 Version 1.0.9rc1 (2016-11-27)
 -----------------------------
 

docs/usage/debug_delete-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_delete-obj:
+
+borg debug delete-obj
+---------------------
+::
+
+    borg debug delete-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to delete from the repo
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command deletes objects from the repository.

docs/usage/debug_dump-archive-items.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-archive-items:
+
+borg debug dump-archive-items
+-----------------------------
+::
+
+    borg debug dump-archive-items <options> ARCHIVE
+
+positional arguments
+    ARCHIVE
+        archive to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) archive items (only metadata) to files.

docs/usage/debug_dump-repo-objs.rst.inc

@@ -0,0 +1,21 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_dump-repo-objs:
+
+borg debug dump-repo-objs
+-------------------------
+::
+
+    borg debug dump-repo-objs <options> REPOSITORY
+
+positional arguments
+    REPOSITORY
+        repo to dump
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command dumps raw (but decrypted and decompressed) repo objects to files.

docs/usage/debug_get-obj.rst.inc

@@ -0,0 +1,25 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_get-obj:
+
+borg debug get-obj
+------------------
+::
+
+    borg debug get-obj <options> REPOSITORY ID PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    ID
+        hex object ID to get from the repo
+    PATH
+        file to write object data into
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command gets an object from the repository.

docs/usage/debug_info.rst.inc

@@ -0,0 +1,19 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_info:
+
+borg debug info
+---------------
+::
+
+    borg debug info <options>
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays some system information that might be useful for bug
+reports and debugging problems. If a traceback happens, this information is
+already appended at the end of the traceback.

docs/usage/debug_put-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_put-obj:
+
+borg debug put-obj
+------------------
+::
+
+    borg debug put-obj <options> REPOSITORY PATH
+
+positional arguments
+    REPOSITORY
+        repository to use
+    PATH
+        file(s) to read and create object(s) from
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command puts objects into the repository.

docs/usage/debug_refcount-obj.rst.inc

@@ -0,0 +1,23 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_debug_refcount-obj:
+
+borg debug refcount-obj
+-----------------------
+::
+
+    borg debug refcount-obj <options> REPOSITORY IDs
+
+positional arguments
+    REPOSITORY
+        repository to use
+    IDs
+        hex object ID(s) to show refcounts for
+
+`Common options`_
+    |
+
+Description
+~~~~~~~~~~~
+
+This command displays the reference count for objects from the repository.

setup.py

@@ -226,11 +226,14 @@ def generate_level(self, prefix, parser, Archiver):
             return
         print('found commands: %s' % list(choices.keys()))
 
-        for command, parser in choices.items():
+        for command, parser in sorted(choices.items()):
+            if command.startswith('debug'):
+                print('skipping', command)
+                continue
             print('generating help for %s' % command)
 
             if self.generate_level(command + " ", parser, Archiver):
-                break
+                continue
 
             with open('docs/usage/%s.rst.inc' % command.replace(" ", "_"), 'w') as doc:
                 doc.write(".. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!\n\n")

src/borg/archiver.py

@@ -1485,6 +1485,11 @@ def do_help(self, parser, commands, args):
             parser.error('No help available on %s' % (args.topic,))
         return self.exit_code
 
+    def do_subcommand_help(self, parser, args):
+        """display infos about subcommand"""
+        parser.print_help()
+        return EXIT_SUCCESS
+
     def preprocess_args(self, args):
         deprecations = [
             # ('--old', '--new', 'Warning: "--old" has been deprecated. Use "--new" instead.'),
@@ -1723,13 +1728,14 @@ def build_parser(self, prog=None):
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
                                type=location_validator(archive=False))
 
-        subparser = subparsers.add_parser('key', add_help=True,
+        subparser = subparsers.add_parser('key', parents=[common_parser], add_help=False,
                                           description="Manage a keyfile or repokey of a repository",
                                           epilog="",
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='manage repository key')
 
         key_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         key_export_epilog = textwrap.dedent("""
         If repository encryption is used, the repository is inaccessible
@@ -2512,13 +2518,14 @@ def build_parser(self, prog=None):
         in case you ever run into some severe malfunction. Use them only if you know
         what you are doing or if a trusted developer tells you what to do.""")
 
-        subparser = subparsers.add_parser('debug', add_help=True,
+        subparser = subparsers.add_parser('debug', parents=[common_parser], add_help=False,
                                           description='debugging command (not intended for normal use)',
                                           epilog=debug_epilog,
                                           formatter_class=argparse.RawDescriptionHelpFormatter,
                                           help='debugging command (not intended for normal use)')
 
         debug_parsers = subparser.add_subparsers(title='required arguments', metavar='<command>')
+        subparser.set_defaults(func=functools.partial(self.do_subcommand_help, subparser))
 
         debug_info_epilog = textwrap.dedent("""
         This command displays some system information that might be useful for bug

src/borg/helpers.py

@@ -889,6 +889,17 @@ class Location:
     """
     proto = user = host = port = path = archive = None
 
+    # user must not contain "@", ":" or "/".
+    # Quoting adduser error message:
+    # "To avoid problems, the username should consist only of letters, digits,
+    # underscores, periods, at signs and dashes, and not start with a dash
+    # (as defined by IEEE Std 1003.1-2001)."
+    # We use "@" as separator between username and hostname, so we must
+    # disallow it within the pure username part.
+    optional_user_re = r"""
+        (?:(?P<user>[^@:/]+)@)?
+    """
+
     # path must not contain :: (it ends at :: or string end), but may contain single colons.
     # to avoid ambiguities with other regexes, it must also not start with ":".
     path_re = r"""
@@ -907,7 +918,7 @@ class Location:
     # regexes for misc. kinds of supported location specifiers:
     ssh_re = re.compile(r"""
         (?P<proto>ssh)://                                   # ssh://
-        (?:(?P<user>[^@]+)@)?                               # user@  (optional)
+        """ + optional_user_re + r"""                       # user@  (optional)
         (?P<host>[^:/]+)(?::(?P<port>\d+))?                 # host or host:port
         """ + path_re + optional_archive_re, re.VERBOSE)    # path or path::archive
 
@@ -918,7 +929,7 @@ class Location:
     # note: scp_re is also use for local pathes
     scp_re = re.compile(r"""
         (
-            (?:(?P<user>[^@]+)@)?                           # user@  (optional)
+            """ + optional_user_re + r"""                   # user@  (optional)
             (?P<host>[^:/]+):                               # host: (don't match / in host to disambiguate from file:)
         )?                                                  # user@host: part is optional
         """ + path_re + optional_archive_re, re.VERBOSE)    # path with optional archive

src/borg/testsuite/helpers.py

@@ -97,6 +97,13 @@ def test_with_colons(self, monkeypatch):
         assert repr(Location('/abs/path:with:colons')) == \
             "Location(proto='file', user=None, host=None, port=None, path='/abs/path:with:colons', archive=None)"
 
+    def test_user_parsing(self):
+        # see issue #1930
+        assert repr(Location('host:path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='path', archive='2016-12-31@23:59:59')"
+        assert repr(Location('ssh://host/path::2016-12-31@23:59:59')) == \
+            "Location(proto='ssh', user=None, host='host', port=None, path='/path', archive='2016-12-31@23:59:59')"
+
     def test_underspecified(self, monkeypatch):
         monkeypatch.delenv('BORG_REPO', raising=False)
         with pytest.raises(ValueError):