From 48fa449e39c3984303e9190c1e85050abb0fcffb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 23 Oct 2016 20:36:05 +0200 Subject: [PATCH 01/12] assert_dirs_equal: add ignore_bsdflags and ignore_xattrs argument bsdflags are not supported in the FUSE mount. xattrs are supported, but are tested separately. --- borg/testsuite/__init__.py | 15 ++++++++------- borg/testsuite/archiver.py | 11 ++++++++--- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 8e09771d7..42b935d1e 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -60,11 +60,11 @@ def assert_creates_file(self, path): yield self.assert_true(os.path.exists(path), '{} should exist'.format(path)) - def assert_dirs_equal(self, dir1, dir2): + def assert_dirs_equal(self, dir1, dir2, **kwargs): diff = filecmp.dircmp(dir1, dir2) - self._assert_dirs_equal_cmp(diff) + self._assert_dirs_equal_cmp(diff, **kwargs) - def _assert_dirs_equal_cmp(self, diff): + def _assert_dirs_equal_cmp(self, diff, ignore_bsdflags=False, ignore_xattrs=False): self.assert_equal(diff.left_only, []) self.assert_equal(diff.right_only, []) self.assert_equal(diff.diff_files, []) @@ -77,7 +77,7 @@ def _assert_dirs_equal_cmp(self, diff): # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev'] - if has_lchflags: + if has_lchflags and not ignore_bsdflags: attrs.append('st_flags') if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our fuse filesystem @@ -96,11 +96,12 @@ def _assert_dirs_equal_cmp(self, diff): else: d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) - d1.append(no_selinux(get_all(path1, follow_symlinks=False))) - d2.append(no_selinux(get_all(path2, follow_symlinks=False))) + if not ignore_xattrs: + d1.append(no_selinux(get_all(path1, follow_symlinks=False))) + d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): - self._assert_dirs_equal_cmp(sub_diff) + self._assert_dirs_equal_cmp(sub_diff, ignore_bsdflags=ignore_bsdflags, ignore_xattrs=ignore_xattrs) @contextmanager def fuse_mount(self, location, mountpoint, *options): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index ee68f61b1..cbda687b5 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1040,11 +1040,16 @@ def has_noatime(some_file): mountpoint = os.path.join(self.tmpdir, 'mountpoint') # mount the whole repository, archive contents shall show up in archivename subdirs of mountpoint: with self.fuse_mount(self.repository_location, mountpoint): - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input')) - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input')) + # bsdflags are not supported by the FUSE mount + # we also ignore xattrs here, they are tested separately + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive', 'input'), + ignore_bsdflags=True, ignore_xattrs=True) + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'archive2', 'input'), + ignore_bsdflags=True, ignore_xattrs=True) # mount only 1 archive, its contents shall show up directly in mountpoint: with self.fuse_mount(self.repository_location + '::archive', mountpoint): - self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input')) + self.assert_dirs_equal(self.input_path, os.path.join(mountpoint, 'input'), + ignore_bsdflags=True, ignore_xattrs=True) # regular file in_fn = 'input/file1' out_fn = os.path.join(mountpoint, 'input', 'file1') From ede3b4a3549cc1f71e47a2a0e213d94a55e6135d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 24 Oct 2016 00:49:03 +0200 Subject: [PATCH 02/12] fuse: test troublesome xattrs last --- borg/testsuite/archiver.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index cbda687b5..3a1478563 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1069,20 +1069,6 @@ def has_noatime(some_file): # read with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f: assert in_f.read() == out_f.read() - # list/read xattrs - in_fn = 'input/fusexattr' - out_fn = os.path.join(mountpoint, 'input', 'fusexattr') - if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] - assert xattr.getxattr(out_fn, 'user.foo') == b'bar' - else: - assert xattr.listxattr(out_fn) == [] - try: - xattr.getxattr(out_fn, 'user.foo') - except OSError as e: - assert e.errno == llfuse.ENOATTR - else: - assert False, "expected OSError(ENOATTR), but no error was raised" # hardlink (to 'input/file1') in_fn = 'input/hardlink' out_fn = os.path.join(mountpoint, 'input', 'hardlink') @@ -1102,6 +1088,20 @@ def has_noatime(some_file): out_fn = os.path.join(mountpoint, 'input', 'fifo1') sto = os.stat(out_fn) assert stat.S_ISFIFO(sto.st_mode) + # list/read xattrs + in_fn = 'input/fusexattr' + out_fn = os.path.join(mountpoint, 'input', 'fusexattr') + if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): + assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] + assert xattr.getxattr(out_fn, 'user.foo') == b'bar' + else: + assert xattr.listxattr(out_fn) == [] + try: + xattr.getxattr(out_fn, 'user.foo') + except OSError as e: + assert e.errno == llfuse.ENOATTR + else: + assert False, "expected OSError(ENOATTR), but no error was raised" @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): From 02ecf047801e515c6ba57f8f1560cf6d354aa379 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 24 Oct 2016 01:19:36 +0200 Subject: [PATCH 03/12] fuse tests: catch ENOTSUP on freebsd seems like fuse does not support xattrs there at all. --- borg/testsuite/archiver.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 3a1478563..3cd317b72 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1089,19 +1089,26 @@ def has_noatime(some_file): sto = os.stat(out_fn) assert stat.S_ISFIFO(sto.st_mode) # list/read xattrs - in_fn = 'input/fusexattr' - out_fn = os.path.join(mountpoint, 'input', 'fusexattr') - if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] - assert xattr.getxattr(out_fn, 'user.foo') == b'bar' - else: - assert xattr.listxattr(out_fn) == [] - try: - xattr.getxattr(out_fn, 'user.foo') - except OSError as e: - assert e.errno == llfuse.ENOATTR + try: + in_fn = 'input/fusexattr' + out_fn = os.path.join(mountpoint, 'input', 'fusexattr') + if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): + assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] + assert xattr.getxattr(out_fn, 'user.foo') == b'bar' else: - assert False, "expected OSError(ENOATTR), but no error was raised" + assert xattr.listxattr(out_fn) == [] + try: + xattr.getxattr(out_fn, 'user.foo') + except OSError as e: + assert e.errno == llfuse.ENOATTR + else: + assert False, "expected OSError(ENOATTR), but no error was raised" + except OSError as err: + if sys.platform.startswith(('freebsd', )) and err.errno == errno.ENOTSUP: + # some systems have no xattr support on FUSE + pass + else: + raise @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): From baa77c0e0483a6e09035462dcfaee4bde2a90d06 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 18 Oct 2016 21:36:23 +0200 Subject: [PATCH 04/12] avoid previous_location mismatch, fixes #1741 due to the changed canonicalization for relative pathes in PR #1711 (implement /./ relpath hack), there would be a changed repo location warning and the user would be asked if this is ok. this would break automation and require manual intervention, which is unwanted. thus, we automatically fix the previous_location config entry, if it only changed in the expected way, but still means the same location. --- borg/cache.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/borg/cache.py b/borg/cache.py index 43ef5c94b..53fdae7ca 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -10,7 +10,7 @@ from .logger import create_logger logger = create_logger() from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \ - bigint_to_int, format_file_size, yes, bin_to_hex + bigint_to_int, format_file_size, yes, bin_to_hex, Location from .locking import Lock from .hashindex import ChunkIndex @@ -140,10 +140,7 @@ def create(self): with open(os.path.join(self.path, 'files'), 'wb') as fd: pass # empty file - def _do_open(self): - self.config = configparser.ConfigParser(interpolation=None) - config_path = os.path.join(self.path, 'config') - self.config.read(config_path) + def _check_upgrade(self, config_path): try: cache_version = self.config.getint('cache', 'version') wanted_version = 1 @@ -154,6 +151,25 @@ def _do_open(self): except configparser.NoSectionError: self.close() raise Exception('%s does not look like a Borg cache.' % config_path) from None + # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741). + cache_loc = self.config.get('cache', 'previous_location', fallback=None) + if cache_loc: + repo_loc = self.repository._location.canonical_path() + rl = Location(repo_loc) + cl = Location(cache_loc) + if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \ + and \ + cl.path and rl.path and \ + cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]: + # everything is same except the expected change in relative path canonicalization, + # update previous_location to avoid warning / user query about changed location: + self.config.set('cache', 'previous_location', repo_loc) + + def _do_open(self): + self.config = configparser.ConfigParser(interpolation=None) + config_path = os.path.join(self.path, 'config') + self.config.read(config_path) + self._check_upgrade(config_path) self.id = self.config.get('cache', 'repository') self.manifest_id = unhexlify(self.config.get('cache', 'manifest')) self.timestamp = self.config.get('cache', 'timestamp', fallback=None) From 2df8b740ddd00827ff07246e172d054a7a602a02 Mon Sep 17 00:00:00 2001 From: Andrew Skalski Date: Wed, 25 May 2016 18:14:27 -0400 Subject: [PATCH 05/12] RemoteRepository: Fix busy wait in call_many, fixes #940 (cherry picked from commit 731f6241faa254c379a9bc25dd21133d3604d32a) --- borg/remote.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index e04802314..00c3114a4 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -20,6 +20,8 @@ BUFSIZE = 10 * 1024 * 1024 +MAX_INFLIGHT = 100 + class ConnectionClosed(Error): """Connection closed by remote host""" @@ -316,7 +318,6 @@ def handle_error(error, res): calls = list(calls) waiting_for = [] - w_fds = [self.stdin_fd] while wait or calls: while waiting_for: try: @@ -330,6 +331,10 @@ def handle_error(error, res): return except KeyError: break + if self.to_send or ((calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT): + w_fds = [self.stdin_fd] + else: + w_fds = [] r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) if x: raise Exception('FD exception occurred') @@ -362,7 +367,7 @@ def handle_error(error, res): else: sys.stderr.write("Remote: " + line) if w: - while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < 100: + while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: if calls: if is_preloaded: if calls[0] in self.cache: @@ -389,8 +394,6 @@ def handle_error(error, res): # that the fd should be writable if e.errno != errno.EAGAIN: raise - if not self.to_send and not (calls or self.preload_ids): - w_fds = [] self.ignore_responses |= set(waiting_for) def check(self, repair=False, save_space=False): From 9b9179312d0c471746f8c0e21a0dbd5b5ddffe92 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 28 Oct 2016 03:32:37 +0200 Subject: [PATCH 06/12] ssh: repo url docs - fix typo --- docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f7105d6a7..d2e803f0d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -249,7 +249,7 @@ is installed on the remote host, in which case the following syntax is used:: or:: - $ borg init ssh://user@hostname:port//path/to/repo + $ borg init ssh://user@hostname:port/path/to/repo Remote operations over SSH can be automated with SSH keys. You can restrict the use of the SSH keypair by prepending a forced command to the SSH public key in From ca15cc80e52a89dabdd7cf45b6ca609d039e7f3d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 28 Oct 2016 04:36:38 +0200 Subject: [PATCH 07/12] document repo URLs / archive location --- docs/usage.rst | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index d9fcc3a22..e72f3cb0b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -12,6 +12,76 @@ command in detail. General ------- +Repository URLs +~~~~~~~~~~~~~~~ + +**Local filesystem** (or locally mounted network filesystem): + +``/path/to/repo`` - filesystem path to repo directory, absolute path + +``path/to/repo`` - filesystem path to repo directory, relative path + +Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is +expanded by your shell). + + +**Remote repositories** accessed via ssh user@host: + +``user@host:/path/to/repo`` - remote repo, absolute path + +``ssh://user@host:port/path/to/repo`` - same, alternative syntax, port can be given + + +**Remote repositories with relative pathes** can be given using this syntax: + +``user@host:path/to/repo`` - path relative to current directory + +``user@host:~/path/to/repo`` - path relative to user's home directory + +``user@host:~other/path/to/repo`` - path relative to other's home directory + +Note: giving ``user@host:/./path/to/repo`` or ``user@host:/~/path/to/repo`` or +``user@host:/~other/path/to/repo``is also supported, but not required here. + + +**Remote repositories with relative pathes, alternative syntax with port**: + +``ssh://user@host:port/./path/to/repo`` - path relative to current directory + +``ssh://user@host:port/~/path/to/repo`` - path relative to user's home directory + +``ssh://user@host:port/~other/path/to/repo`` - path relative to other's home directory + + +If you frequently need the same repo URL, it is a good idea to set the +``BORG_REPO`` environment variable to set a default for the repo URL: + +:: + + export BORG_REPO='ssh://user@host:port/path/to/repo' + +Then just leave away the repo URL if only a repo URL is needed and you want +to use the default - it will be read from BORG_REPO then. + +Use ``::`` syntax to give the repo URL when syntax requires giving a positional +argument for the repo (e.g. ``borg mount :: /mnt``). + + +Repository / Archive Locations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Many commands want either a repository (just give the repo URL, see above) or +an archive location, which is a repo URL followed by ``::archive_name``. + +Archive names must not contain the ``/`` (slash) character. For simplicity, +maybe also avoid blanks or other characters that have special meaning on the +shell or in a filesystem (borg mount will use the archive name as directory +name). + +If you have set BORG_REPO (see above) and an archive location is needed, use +``::archive_name`` - the repo URL part is then read from BORG_REPO. + + Type of log output ~~~~~~~~~~~~~~~~~~ From e0298b293212a243c53486cb99f0c430225dc6d2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 28 Oct 2016 04:51:46 +0200 Subject: [PATCH 08/12] simplify quickstart only give one possible ssh url syntax, all others are documented in usage chapter. --- docs/quickstart.rst | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index d2e803f0d..75d14bca3 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -247,9 +247,7 @@ is installed on the remote host, in which case the following syntax is used:: $ borg init user@hostname:/path/to/repo -or:: - - $ borg init ssh://user@hostname:port/path/to/repo +Note: please see the usage chapter for a full documentation of repo URLs. Remote operations over SSH can be automated with SSH keys. You can restrict the use of the SSH keypair by prepending a forced command to the SSH public key in From 11e97803938a6766b298238790f81de4480840c2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 28 Oct 2016 04:57:15 +0200 Subject: [PATCH 09/12] quickstart: add a comment about other (remote) filesystems --- docs/quickstart.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 75d14bca3..26387aecf 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -263,3 +263,7 @@ mounting the remote filesystem, for example, using sshfs:: $ sshfs user@hostname:/path/to /path/to $ borg init /path/to/repo $ fusermount -u /path/to + +You can also use other remote filesystems in a similar way. Just be careful, +not all filesystems out there are really stable and working good enough to +be acceptable for backup usage. From a16c7d8e16c472d05034d2c5350283f95d86db19 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 28 Oct 2016 05:04:23 +0200 Subject: [PATCH 10/12] mention file:// --- docs/usage.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/usage.rst b/docs/usage.rst index e72f3cb0b..8743f6ac0 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -24,6 +24,7 @@ Repository URLs Also, stuff like ``~/path/to/repo`` or ``~other/path/to/repo`` works (this is expanded by your shell). +Note: you may also prepend a ``file://`` to a filesystem path to get URL style. **Remote repositories** accessed via ssh user@host: From 20f4a1f4785920a394374401d5ee5b612805b845 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 25 Oct 2016 21:23:23 +0200 Subject: [PATCH 11/12] update CHANGES (1.0-maint) --- docs/changes.rst | 48 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 8f8dc9a87..70c1d3726 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -50,6 +50,48 @@ The best check that everything is ok is to run a dry-run extraction:: borg extract -v --dry-run REPO::ARCHIVE +Version 1.0.8 (2016-10-29) +-------------------------- + +Bug fixes: + +- RemoteRepository: Fix busy wait in call_many, #940 + +New features: + +- implement borgmajor/borgminor/borgpatch placeholders, #1694 + {borgversion} was already there (full version string). With the new + placeholders you can now also get e.g. 1 or 1.0 or 1.0.8. + +Other changes: + +- avoid previous_location mismatch, #1741 + + due to the changed canonicalization for relative pathes in PR #1711 / #1655 + (implement /./ relpath hack), there would be a changed repo location warning + and the user would be asked if this is ok. this would break automation and + require manual intervention, which is unwanted. + + thus, we automatically fix the previous_location config entry, if it only + changed in the expected way, but still means the same location. + +- docs: + + - deployment.rst: do not use bare variables in ansible snippet + - add clarification about append-only mode, #1689 + - setup.py: add comment about requiring llfuse, #1726 + - update usage.rst / api.rst + - repo url / archive location docs + typo fix + - quickstart: add a comment about other (remote) filesystems + +- vagrant / tests: + + - no chown when rsyncing (fixes boxes w/o vagrant group) + - fix fuse permission issues on linux/freebsd, #1544 + - skip fuse test for borg binary + fakeroot + - ignore security.selinux xattrs, fixes tests on centos, #1735 + + Version 1.0.8rc1 (2016-10-17) ----------------------------- @@ -72,8 +114,8 @@ Bug fixes: (this seems not to get triggered in 1.0.x, but was discovered in master) - hashindex: fix iterators (always raise StopIteration when exhausted) (this seems not to get triggered in 1.0.x, but was discovered in master) -- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655 -- allow repo pathes with colons, fixes #1705 +- enable relative pathes in ssh:// repo URLs, via /./relpath hack, #1655 +- allow repo pathes with colons, #1705 - update changed repo location immediately after acceptance, #1524 - fix debug get-obj / delete-obj crash if object not found and remote repo, #1684 @@ -105,7 +147,7 @@ Other changes: appears not only in the traceback, but also in the (short) error message, #1572 - borg.key: include chunk id in exception msgs, #1571 - - better messages for cache newer than repo, fixes #1700 + - better messages for cache newer than repo, #1700 - vagrant (testing/build VMs): - upgrade OSXfuse / FUSE for macOS to 3.5.2 From 84637322f29ec7a54e16440fe92ab76bfe113510 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 29 Oct 2016 01:58:21 +0200 Subject: [PATCH 12/12] ran build_usage --- docs/usage/help.rst.inc | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index b079dd2d6..b1025fb5b 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -128,7 +128,19 @@ borg help placeholders {borgversion} - The version of borg. + The version of borg, e.g.: 1.0.8rc1 + + {borgmajor} + + The version of borg, only the major version, e.g.: 1 + + {borgminor} + + The version of borg, only major and minor version, e.g.: 1.0 + + {borgpatch} + + The version of borg, only major, minor and patch version, e.g.: 1.0.8 Examples::