Merge branch '1.0-maint' into merge-1.0-maint

# Conflicts:
#	docs/changes.rst
#	docs/usage/help.rst.inc
#	src/borg/cache.py
#	src/borg/remote.py
#	src/borg/testsuite/__init__.py
#	src/borg/testsuite/archiver.py

docs/changes.rst

@@ -218,6 +218,48 @@ Other changes:
   - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945
 
 
+Version 1.0.8 (2016-10-29)
+--------------------------
+
+Bug fixes:
+
+- RemoteRepository: Fix busy wait in call_many, #940
+
+New features:
+
+- implement borgmajor/borgminor/borgpatch placeholders, #1694
+  {borgversion} was already there (full version string). With the new
+  placeholders you can now also get e.g. 1 or 1.0 or 1.0.8.
+
+Other changes:
+
+- avoid previous_location mismatch, #1741
+
+  due to the changed canonicalization for relative pathes in PR #1711 / #1655
+  (implement /./ relpath hack), there would be a changed repo location warning
+  and the user would be asked if this is ok. this would break automation and
+  require manual intervention, which is unwanted.
+
+  thus, we automatically fix the previous_location config entry, if it only
+  changed in the expected way, but still means the same location.
+
+- docs:
+
+  - deployment.rst: do not use bare variables in ansible snippet
+  - add clarification about append-only mode, #1689
+  - setup.py: add comment about requiring llfuse, #1726
+  - update usage.rst / api.rst
+  - repo url / archive location docs + typo fix
+  - quickstart: add a comment about other (remote) filesystems
+
+- vagrant / tests:
+
+  - no chown when rsyncing (fixes boxes w/o vagrant group)
+  - fix fuse permission issues on linux/freebsd, #1544
+  - skip fuse test for borg binary + fakeroot
+  - ignore security.selinux xattrs, fixes tests on centos, #1735
+
+
 Version 1.0.8rc1 (2016-10-17)
 -----------------------------
 
@@ -240,8 +282,8 @@ Bug fixes:
   (this seems not to get triggered in 1.0.x, but was discovered in master)
 - hashindex: fix iterators (always raise StopIteration when exhausted)
   (this seems not to get triggered in 1.0.x, but was discovered in master)
-- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655
-- allow repo pathes with colons, fixes #1705
+- enable relative pathes in ssh:// repo URLs, via /./relpath hack, #1655
+- allow repo pathes with colons, #1705
 - update changed repo location immediately after acceptance, #1524
 - fix debug get-obj / delete-obj crash if object not found and remote repo,
   #1684
@@ -273,7 +315,7 @@ Other changes:
     appears not only in the traceback, but also in the (short) error message,
     #1572
   - borg.key: include chunk id in exception msgs, #1571
-  - better messages for cache newer than repo, fixes #1700
+  - better messages for cache newer than repo, #1700
 - vagrant (testing/build VMs):
 
   - upgrade OSXfuse / FUSE for macOS to 3.5.2

docs/quickstart.rst

@@ -267,9 +267,7 @@ is installed on the remote host, in which case the following syntax is used::
 
   $ borg init user@hostname:/path/to/repo
 
-or::
-
-  $ borg init ssh://user@hostname:port//path/to/repo
+Note: please see the usage chapter for a full documentation of repo URLs.
 
 Remote operations over SSH can be automated with SSH keys. You can restrict the
 use of the SSH keypair by prepending a forced command to the SSH public key in
@@ -285,3 +283,7 @@ mounting the remote filesystem, for example, using sshfs::
   $ sshfs user@hostname:/path/to /path/to
   $ borg init /path/to/repo
   $ fusermount -u /path/to
+
+You can also use other remote filesystems in a similar way. Just be careful,
+not all filesystems out there are really stable and working good enough to
+be acceptable for backup usage.

docs/usage.rst

@@ -12,6 +12,77 @@ command in detail.
 General
 -------
 
+Repository URLs
+~~~~~~~~~~~~~~~
+
+**Local filesystem** (or locally mounted network filesystem):
+
+``/path/to/repo`` - filesystem path to repo directory, absolute path
+
+``path/to/repo`` - filesystem path to repo directory, relative path
+
+Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is
+expanded by your shell).
+
+Note: you may also prepend a ``file://`` to a filesystem path to get URL style.
+
+**Remote repositories** accessed via ssh user@host:
+
+``user@host:/path/to/repo`` - remote repo, absolute path
+
+``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given
+
+
+**Remote repositories with relative pathes** can be given using this syntax:
+
+``user@host:path/to/repo`` - path relative to current directory
+
+``user@host:~/path/to/repo`` - path relative to user's home directory
+
+``user@host:~other/path/to/repo`` - path relative to other's home directory
+
+Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or
+``user@host:/~other/path/to/repo``is also supported, but not required here.
+
+
+**Remote repositories with relative pathes, alternative syntax with port**:
+
+``ssh://user@host:port/./path/to/repo`` - path relative to current directory
+
+``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory
+
+``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory
+
+
+If you frequently need the same repo URL, it is a good idea to set the
+``BORG_REPO`` environment variable to set a default for the repo URL:
+
+::
+
+    export BORG_REPO='ssh://user@host:port/path/to/repo'
+
+Then just leave away the repo URL if only a repo URL is needed and you want
+to use the default - it will be read from BORG_REPO then.
+
+Use ``::`` syntax to give the repo URL when syntax requires giving a positional
+argument for the repo (e.g. ``borg mount :: /mnt``).
+
+
+Repository / Archive Locations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Many commands want either a repository (just give the repo URL, see above) or
+an archive location, which is a repo URL followed by ``::archive_name``.
+
+Archive names must not contain the ``/`` (slash) character. For simplicity,
+maybe also avoid blanks or other characters that have special meaning on the
+shell or in a filesystem (borg mount will use the archive name as directory
+name).
+
+If you have set BORG_REPO (see above) and an archive location is needed, use
+``::archive_name`` - the repo URL part is then read from BORG_REPO.
+
+
 Type of log output
 ~~~~~~~~~~~~~~~~~~
 

docs/usage/help.rst.inc

@@ -130,7 +130,19 @@ placeholders:
 
 {borgversion}
 
-    The version of borg.
+     The version of borg, e.g.: 1.0.8rc1
+
+ {borgmajor}
+
+     The version of borg, only the major version, e.g.: 1
+
+ {borgminor}
+
+     The version of borg, only major and minor version, e.g.: 1.0
+
+ {borgpatch}
+
+     The version of borg, only major, minor and patch version, e.g.: 1.0.8
 
 Examples::
 

src/borg/cache.py

@@ -11,6 +11,7 @@
 logger = create_logger()
 
 from .hashindex import ChunkIndex, ChunkIndexEntry
+from .helpers import Location
 from .helpers import Error
 from .helpers import get_cache_dir
 from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex
@@ -160,10 +161,7 @@ def create(self):
         with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
             pass  # empty file
 
-    def _do_open(self):
-        self.config = configparser.ConfigParser(interpolation=None)
-        config_path = os.path.join(self.path, 'config')
-        self.config.read(config_path)
+    def _check_upgrade(self, config_path):
         try:
             cache_version = self.config.getint('cache', 'version')
             wanted_version = 1
@@ -174,6 +172,25 @@ def _do_open(self):
         except configparser.NoSectionError:
             self.close()
             raise Exception('%s does not look like a Borg cache.' % config_path) from None
+        # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741).
+        cache_loc = self.config.get('cache', 'previous_location', fallback=None)
+        if cache_loc:
+            repo_loc = self.repository._location.canonical_path()
+            rl = Location(repo_loc)
+            cl = Location(cache_loc)
+            if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
+                    and \
+                    cl.path and rl.path and \
+                    cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
+                # everything is same except the expected change in relative path canonicalization,
+                # update previous_location to avoid warning / user query about changed location:
+                self.config.set('cache', 'previous_location', repo_loc)
+
+    def _do_open(self):
+        self.config = configparser.ConfigParser(interpolation=None)
+        config_path = os.path.join(self.path, 'config')
+        self.config.read(config_path)
+        self._check_upgrade(config_path)
         self.id = self.config.get('cache', 'repository')
         self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
         self.timestamp = self.config.get('cache', 'timestamp', fallback=None)

src/borg/testsuite/__init__.py

@@ -146,11 +146,11 @@ def assert_creates_file(self, path):
         yield
         self.assert_true(os.path.exists(path), '{} should exist'.format(path))
 
-    def assert_dirs_equal(self, dir1, dir2):
+    def assert_dirs_equal(self, dir1, dir2, **kwargs):
         diff = filecmp.dircmp(dir1, dir2)
-        self._assert_dirs_equal_cmp(diff)
+        self._assert_dirs_equal_cmp(diff, **kwargs)
 
-    def _assert_dirs_equal_cmp(self, diff):
+    def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False):
         self.assert_equal(diff.left_only, [])
         self.assert_equal(diff.right_only, [])
         self.assert_equal(diff.diff_files, [])
@@ -168,8 +168,9 @@ def _assert_dirs_equal_cmp(self, diff):
                 attrs.append('st_nlink')
             d1 = [filename] + [getattr(s1, a) for a in attrs]
             d2 = [filename] + [getattr(s2, a) for a in attrs]
-            d1.append(get_flags(path1, s1))
-            d2.append(get_flags(path2, s2))
+            if not ignore_bsdflags:
+                d1.append(get_flags(path1, s1))
+                d2.append(get_flags(path2, s2))
             # ignore st_rdev if file is not a block/char device, fixes #203
             if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]):
                 d1[4] = None
@@ -185,11 +186,12 @@ def _assert_dirs_equal_cmp(self, diff):
                 else:
                     d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
                     d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
-            d1.append(no_selinux(get_all(path1, follow_symlinks=False)))
-            d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
+            if not ignore_xattrs:
+                d1.append(no_selinux(get_all(path1, follow_symlinks=False)))
+                d2.append(no_selinux(get_all(path2, follow_symlinks=False)))
             self.assert_equal(d1, d2)
         for sub_diff in diff.subdirs.values():
-            self._assert_dirs_equal_cmp(sub_diff)
+            self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs)
 
     @contextmanager
     def fuse_mount(self, location, mountpoint, *options):

src/borg/testsuite/archiver.py

@@ -1402,11 +1402,16 @@ def has_noatime(some_file):
         mountpoint = os.path.join(self.tmpdir, 'mountpoint')
         # mount the whole repository, archive contents shall show up in archivename subdirs of mountpoint:
         with self.fuse_mount(self.repository_location, mountpoint):
-            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'))
-            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'))
+            # bsdflags are not supported by the FUSE mount
+            # we also ignore xattrs here, they are tested separately
+            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'),
+                                   ignore_bsdflags=True, ignore_xattrs=True)
+            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'),
+                                   ignore_bsdflags=True, ignore_xattrs=True)
         # mount only 1 archive, its contents shall show up directly in mountpoint:
         with self.fuse_mount(self.repository_location + '::archive', mountpoint):
-            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'))
+            self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'),
+                                   ignore_bsdflags=True, ignore_xattrs=True)
             # regular file
             in_fn = 'input/file1'
             out_fn = os.path.join(mountpoint, 'input', 'file1')
@@ -1426,20 +1431,6 @@ def has_noatime(some_file):
             # read
             with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f:
                 assert in_f.read() == out_f.read()
-            # list/read xattrs
-            in_fn = 'input/fusexattr'
-            out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
-            if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
-                assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ]
-                assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
-            else:
-                assert xattr.listxattr(out_fn) == []
-                try:
-                    xattr.getxattr(out_fn, 'user.foo')
-                except OSError as e:
-                    assert e.errno == llfuse.ENOATTR
-                else:
-                    assert False, "expected OSError(ENOATTR), but no error was raised"
             # hardlink (to 'input/file1')
             if are_hardlinks_supported():
                 in_fn = 'input/hardlink'
@@ -1462,6 +1453,27 @@ def has_noatime(some_file):
                 out_fn = os.path.join(mountpoint, 'input', 'fifo1')
                 sto = os.stat(out_fn)
                 assert stat.S_ISFIFO(sto.st_mode)
+            # list/read xattrs
+            try:
+                in_fn = 'input/fusexattr'
+                out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
+                if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
+                    assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ]
+                    assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
+                else:
+                    assert xattr.listxattr(out_fn) == []
+                    try:
+                        xattr.getxattr(out_fn, 'user.foo')
+                    except OSError as e:
+                        assert e.errno == llfuse.ENOATTR
+                    else:
+                        assert False, "expected OSError(ENOATTR), but no error was raised"
+            except OSError as err:
+                if sys.platform.startswith(('freebsd', )) and err.errno == errno.ENOTSUP:
+                    # some systems have no xattr support on FUSE
+                    pass
+                else:
+                    raise
 
     @unittest.skipUnless(has_llfuse, 'llfuse not installed')
     def test_fuse_versions_view(self):