diff --git a/docs/changes.rst b/docs/changes.rst index edcdcfd5e..52fa39510 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -218,6 +218,48 @@ Other changes: - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 +Version 1.0.8 (2016-10-29) +-------------------------- + +Bug fixes: + +- RemoteRepository: Fix busy wait in call_many, #940 + +New features: + +- implement borgmajor/borgminor/borgpatch placeholders, #1694 + {borgversion} was already there (full version string). With the new + placeholders you can now also get e.g. 1 or 1.0 or 1.0.8. + +Other changes: + +- avoid previous_location mismatch, #1741 + + due to the changed canonicalization for relative pathes in PR #1711 / #1655 + (implement /./ relpath hack), there would be a changed repo location warning + and the user would be asked if this is ok. this would break automation and + require manual intervention, which is unwanted. + + thus, we automatically fix the previous_location config entry, if it only + changed in the expected way, but still means the same location. + +- docs: + + - deployment.rst: do not use bare variables in ansible snippet + - add clarification about append-only mode, #1689 + - setup.py: add comment about requiring llfuse, #1726 + - update usage.rst / api.rst + - repo url / archive location docs + typo fix + - quickstart: add a comment about other (remote) filesystems + +- vagrant / tests: + + - no chown when rsyncing (fixes boxes w/o vagrant group) + - fix fuse permission issues on linux/freebsd, #1544 + - skip fuse test for borg binary + fakeroot + - ignore security.selinux xattrs, fixes tests on centos, #1735 + + Version 1.0.8rc1 (2016-10-17) ----------------------------- @@ -240,8 +282,8 @@ Bug fixes: (this seems not to get triggered in 1.0.x, but was discovered in master) - hashindex: fix iterators (always raise StopIteration when exhausted) (this seems not to get triggered in 1.0.x, but was discovered in master) -- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655 -- allow repo pathes with colons, fixes #1705 +- enable relative pathes in ssh:// repo URLs, via /./relpath hack, #1655 +- allow repo pathes with colons, #1705 - update changed repo location immediately after acceptance, #1524 - fix debug get-obj / delete-obj crash if object not found and remote repo, #1684 @@ -273,7 +315,7 @@ Other changes: appears not only in the traceback, but also in the (short) error message, #1572 - borg.key: include chunk id in exception msgs, #1571 - - better messages for cache newer than repo, fixes #1700 + - better messages for cache newer than repo, #1700 - vagrant (testing/build VMs): - upgrade OSXfuse / FUSE for macOS to 3.5.2 diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 8032db162..3e4e66cb8 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -267,9 +267,7 @@ is installed on the remote host, in which case the following syntax is used:: $ borg init user@hostname:/path/to/repo -or:: - - $ borg init ssh://user@hostname:port//path/to/repo +Note: please see the usage chapter for a full documentation of repo URLs. Remote operations over SSH can be automated with SSH keys. You can restrict the use of the SSH keypair by prepending a forced command to the SSH public key in @@ -285,3 +283,7 @@ mounting the remote filesystem, for example, using sshfs:: $ sshfs user@hostname:/path/to /path/to $ borg init /path/to/repo $ fusermount -u /path/to + +You can also use other remote filesystems in a similar way. Just be careful, +not all filesystems out there are really stable and working good enough to +be acceptable for backup usage. diff --git a/docs/usage.rst b/docs/usage.rst index a2442b9ab..b05b81c48 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -12,6 +12,77 @@ command in detail. General ------- +Repository URLs +~~~~~~~~~~~~~~~ + +**Local filesystem** (or locally mounted network filesystem): + +``/path/to/repo`` - filesystem path to repo directory, absolute path + +``path/to/repo`` - filesystem path to repo directory, relative path + +Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is +expanded by your shell). + +Note: you may also prepend a ``file://`` to a filesystem path to get URL style. + +**Remote repositories** accessed via ssh user@host: + +``user@host:/path/to/repo`` - remote repo, absolute path + +``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given + + +**Remote repositories with relative pathes** can be given using this syntax: + +``user@host:path/to/repo`` - path relative to current directory + +``user@host:~/path/to/repo`` - path relative to user's home directory + +``user@host:~other/path/to/repo`` - path relative to other's home directory + +Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or +``user@host:/~other/path/to/repo``is also supported, but not required here. + + +**Remote repositories with relative pathes, alternative syntax with port**: + +``ssh://user@host:port/./path/to/repo`` - path relative to current directory + +``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory + +``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory + + +If you frequently need the same repo URL, it is a good idea to set the +``BORG_REPO`` environment variable to set a default for the repo URL: + +:: + + export BORG_REPO='ssh://user@host:port/path/to/repo' + +Then just leave away the repo URL if only a repo URL is needed and you want +to use the default - it will be read from BORG_REPO then. + +Use ``::`` syntax to give the repo URL when syntax requires giving a positional +argument for the repo (e.g. ``borg mount :: /mnt``). + + +Repository / Archive Locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many commands want either a repository (just give the repo URL, see above) or +an archive location, which is a repo URL followed by ``::archive_name``. + +Archive names must not contain the ``/`` (slash) character. For simplicity, +maybe also avoid blanks or other characters that have special meaning on the +shell or in a filesystem (borg mount will use the archive name as directory +name). + +If you have set BORG_REPO (see above) and an archive location is needed, use +``::archive_name`` - the repo URL part is then read from BORG_REPO. + + Type of log output ~~~~~~~~~~~~~~~~~~ diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index 572b9f23e..02c2ed9a2 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -130,7 +130,19 @@ placeholders: {borgversion} - The version of borg. + The version of borg, e.g.: 1.0.8rc1 + + {borgmajor} + + The version of borg, only the major version, e.g.: 1 + + {borgminor} + + The version of borg, only major and minor version, e.g.: 1.0 + + {borgpatch} + + The version of borg, only major, minor and patch version, e.g.: 1.0.8 Examples:: diff --git a/src/borg/cache.py b/src/borg/cache.py index fd26e2273..99d5bf233 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -11,6 +11,7 @@ logger = create_logger() from .hashindex import ChunkIndex, ChunkIndexEntry +from .helpers import Location from .helpers import Error from .helpers import get_cache_dir from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex @@ -160,10 +161,7 @@ def create(self): with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd: pass # empty file - def _do_open(self): - self.config = configparser.ConfigParser(interpolation=None) - config_path = os.path.join(self.path, 'config') - self.config.read(config_path) + def _check_upgrade(self, config_path): try: cache_version = self.config.getint('cache', 'version') wanted_version = 1 @@ -174,6 +172,25 @@ def _do_open(self): except configparser.NoSectionError: self.close() raise Exception('%s does not look like a Borg cache.' % config_path) from None + # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741). + cache_loc = self.config.get('cache', 'previous_location', fallback=None) + if cache_loc: + repo_loc = self.repository._location.canonical_path() + rl = Location(repo_loc) + cl = Location(cache_loc) + if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \ + and \ + cl.path and rl.path and \ + cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]: + # everything is same except the expected change in relative path canonicalization, + # update previous_location to avoid warning / user query about changed location: + self.config.set('cache', 'previous_location', repo_loc) + + def _do_open(self): + self.config = configparser.ConfigParser(interpolation=None) + config_path = os.path.join(self.path, 'config') + self.config.read(config_path) + self._check_upgrade(config_path) self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) self.timestamp = self.config.get('cache', 'timestamp', fallback=None) diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index ddcb443a0..0abe8efd2 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -146,11 +146,11 @@ def assert_creates_file(self, path): yield self.assert_true(os.path.exists(path), '{} should exist'.format(path)) - def assert_dirs_equal(self, dir1, dir2): + def assert_dirs_equal(self, dir1, dir2, **kwargs): diff = filecmp.dircmp(dir1, dir2) - self._assert_dirs_equal_cmp(diff) + self._assert_dirs_equal_cmp(diff, **kwargs) - def _assert_dirs_equal_cmp(self, diff): + def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False): self.assert_equal(diff.left_only, []) self.assert_equal(diff.right_only, []) self.assert_equal(diff.diff_files, []) @@ -168,8 +168,9 @@ def _assert_dirs_equal_cmp(self, diff): attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] - d1.append(get_flags(path1, s1)) - d2.append(get_flags(path2, s2)) + if not ignore_bsdflags: + d1.append(get_flags(path1, s1)) + d2.append(get_flags(path2, s2)) # ignore st_rdev if file is not a block/char device, fixes #203 if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]): d1[4] = None @@ -185,11 +186,12 @@ def _assert_dirs_equal_cmp(self, diff): else: d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) - d1.append(no_selinux(get_all(path1, follow_symlinks=False))) - d2.append(no_selinux(get_all(path2, follow_symlinks=False))) + if not ignore_xattrs: + d1.append(no_selinux(get_all(path1, follow_symlinks=False))) + d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): - self._assert_dirs_equal_cmp(sub_diff) + self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs) @contextmanager def fuse_mount(self, location, mountpoint, *options): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index d1db84ce9..5996522ea 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1402,11 +1402,16 @@ def has_noatime(some_file): mountpoint = os.path.join(self.tmpdir, 'mountpoint') # mount the whole repository, archive contents shall show up in archivename subdirs of mountpoint: with self.fuse_mount(self.repository_location, mountpoint): - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input')) - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input')) + # bsdflags are not supported by the FUSE mount + # we also ignore xattrs here, they are tested separately + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'), + ignore_bsdflags=True, ignore_xattrs=True) + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'), + ignore_bsdflags=True, ignore_xattrs=True) # mount only 1 archive, its contents shall show up directly in mountpoint: with self.fuse_mount(self.repository_location + '::archive', mountpoint): - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input')) + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'), + ignore_bsdflags=True, ignore_xattrs=True) # regular file in_fn = 'input/file1' out_fn = os.path.join(mountpoint, 'input', 'file1') @@ -1426,20 +1431,6 @@ def has_noatime(some_file): # read with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f: assert in_f.read() == out_f.read() - # list/read xattrs - in_fn = 'input/fusexattr' - out_fn = os.path.join(mountpoint, 'input', 'fusexattr') - if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] - assert xattr.getxattr(out_fn, 'user.foo') == b'bar' - else: - assert xattr.listxattr(out_fn) == [] - try: - xattr.getxattr(out_fn, 'user.foo') - except OSError as e: - assert e.errno == llfuse.ENOATTR - else: - assert False, "expected OSError(ENOATTR), but no error was raised" # hardlink (to 'input/file1') if are_hardlinks_supported(): in_fn = 'input/hardlink' @@ -1462,6 +1453,27 @@ def has_noatime(some_file): out_fn = os.path.join(mountpoint, 'input', 'fifo1') sto = os.stat(out_fn) assert stat.S_ISFIFO(sto.st_mode) + # list/read xattrs + try: + in_fn = 'input/fusexattr' + out_fn = os.path.join(mountpoint, 'input', 'fusexattr') + if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): + assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] + assert xattr.getxattr(out_fn, 'user.foo') == b'bar' + else: + assert xattr.listxattr(out_fn) == [] + try: + xattr.getxattr(out_fn, 'user.foo') + except OSError as e: + assert e.errno == llfuse.ENOATTR + else: + assert False, "expected OSError(ENOATTR), but no error was raised" + except OSError as err: + if sys.platform.startswith(('freebsd', )) and err.errno == errno.ENOTSUP: + # some systems have no xattr support on FUSE + pass + else: + raise @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_versions_view(self):