From 5d22078f3592e6b3914ec037a8dc86c5572cd868 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Sep 2016 14:56:11 +0200 Subject: [PATCH 1/5] fuse: add parameter check to ItemCache.get to make potential failures more clear --- src/borg/fuse.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 47b8568b1..8e3a8e4da 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -43,7 +43,10 @@ class ItemCache: return pos + self.offset def get(self, inode): - self.fd.seek(inode - self.offset, io.SEEK_SET) + offset = inode - self.offset + if offset < 0: + raise ValueError('ItemCache.get() called with an invalid inode number') + self.fd.seek(offset, io.SEEK_SET) item = next(msgpack.Unpacker(self.fd, read_size=1024)) return Item(internal_dict=item) From 001500ab99ed68c6ed4e277956fcc54225106a0a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Sep 2016 15:41:15 +0200 Subject: [PATCH 2/5] fuse: refactor file versioning code --- src/borg/fuse.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 8e3a8e4da..a8ad62dc3 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -162,17 +162,18 @@ class FuseOperations(llfuse.Operations): self.process_leaf(segments[-1], item, parent, prefix, is_dir) def process_leaf(self, name, item, parent, prefix, is_dir): - def version_name(name, item): + def file_version(item): if 'chunks' in item: ident = 0 for chunkid, _, _ in item.chunks: ident = adler32(chunkid, ident) - name = name + safe_encode('.%08x' % ident) - return name + return ident if self.versions and not is_dir: parent = self.process_inner(name, parent) - name = version_name(name, item) + version = file_version(item) + if version is not None: + name += safe_encode('.%08x' % version) if 'source' in item and stat.S_ISREG(item.mode): inode = self._find_inode(item.source, prefix) From c021cf466a6380271fadfaae8e4dbea57d5b04b7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Sep 2016 20:29:14 +0200 Subject: [PATCH 3/5] fuse: add test for hardlinks in versions view --- src/borg/testsuite/archiver.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index aec0cff60..79777ec52 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1453,6 +1453,9 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_fuse_versions_view(self): self.cmd('init', self.repository_location) self.create_regular_file('test', contents=b'first') + if are_hardlinks_supported(): + self.create_regular_file('hardlink1', contents=b'') + os.link('input/hardlink1', 'input/hardlink2') self.cmd('create', self.repository_location + '::archive1', 'input') self.create_regular_file('test', contents=b'second') self.cmd('create', self.repository_location + '::archive2', 'input') @@ -1463,6 +1466,10 @@ class ArchiverTestCase(ArchiverTestCaseBase): files = os.listdir(path) assert all(f.startswith('test.') for f in files) # ... with files test.xxxxxxxx in there assert {b'first', b'second'} == {open(os.path.join(path, f), 'rb').read() for f in files} + if are_hardlinks_supported(): + st1 = os.stat(os.path.join(mountpoint, 'input', 'hardlink1', 'hardlink1.00000000')) + st2 = os.stat(os.path.join(mountpoint, 'input', 'hardlink2', 'hardlink2.00000000')) + assert st1.st_ino == st2.st_ino @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): From 39170479aa8aff98d031224ca441cab8840c1642 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Sep 2016 20:31:57 +0200 Subject: [PATCH 4/5] fuse: fix hardlinks in versions view, fixes #1599 --- src/borg/fuse.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index a8ad62dc3..f20ae899c 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -15,7 +15,7 @@ from .logger import create_logger logger = create_logger() from .archive import Archive -from .helpers import daemonize, safe_encode +from .helpers import daemonize from .item import Item from .lrucache import LRUCache @@ -136,6 +136,7 @@ class FuseOperations(llfuse.Operations): def process_archive(self, archive, prefix=[]): """Build fuse inode hierarchy from archive metadata """ + self.file_versions = {} # for versions mode: original path -> version unpacker = msgpack.Unpacker() for key, chunk in zip(archive.metadata.items, self.repository.get_many(archive.metadata.items)): _, data = self.key.decrypt(key, chunk) @@ -155,7 +156,6 @@ class FuseOperations(llfuse.Operations): self.items[inode] = item continue segments = prefix + os.fsencode(os.path.normpath(item.path)).split(b'/') - del item.path parent = 1 for segment in segments[:-1]: parent = self.process_inner(segment, parent) @@ -169,14 +169,31 @@ class FuseOperations(llfuse.Operations): ident = adler32(chunkid, ident) return ident + def make_versioned_name(name, version, add_dir=False): + if add_dir: + # add intermediate directory with same name as filename + path_fname = name.rsplit(b'/', 1) + name += b'/' + path_fname[-1] + return name + os.fsencode('.%08x' % version) + if self.versions and not is_dir: parent = self.process_inner(name, parent) version = file_version(item) if version is not None: - name += safe_encode('.%08x' % version) + # regular file, with contents - maybe a hardlink master + name = make_versioned_name(name, version) + self.file_versions[item.path] = version + del item.path # safe some space if 'source' in item and stat.S_ISREG(item.mode): - inode = self._find_inode(item.source, prefix) + # a hardlink, no contents, is the hardlink master + source = item.source + if self.versions: + # adjust source name with version + version = self.file_versions[source] + source = os.fsdecode(make_versioned_name(os.fsencode(source), version, add_dir=True)) + name = make_versioned_name(name, version) + inode = self._find_inode(source, prefix) item = self.cache.get(inode) item.nlink = item.get('nlink', 1) + 1 self.items[inode] = item From 260ef31728a48dce87cd0e0a6a5671c3686ba909 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 15 Sep 2016 21:11:23 +0200 Subject: [PATCH 5/5] fuse: refactor for less encoding dance --- src/borg/fuse.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/borg/fuse.py b/src/borg/fuse.py index f20ae899c..b822332dd 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -143,19 +143,20 @@ class FuseOperations(llfuse.Operations): unpacker.feed(data) for item in unpacker: item = Item(internal_dict=item) + path = os.fsencode(os.path.normpath(item.path)) is_dir = stat.S_ISDIR(item.mode) if is_dir: try: # This can happen if an archive was created with a command line like # $ borg create ... dir1/file dir1 # In this case the code below will have created a default_dir inode for dir1 already. - inode = self._find_inode(item.path, prefix) + inode = self._find_inode(path, prefix) except KeyError: pass else: self.items[inode] = item continue - segments = prefix + os.fsencode(os.path.normpath(item.path)).split(b'/') + segments = prefix + path.split(b'/') parent = 1 for segment in segments[:-1]: parent = self.process_inner(segment, parent) @@ -182,16 +183,17 @@ class FuseOperations(llfuse.Operations): if version is not None: # regular file, with contents - maybe a hardlink master name = make_versioned_name(name, version) - self.file_versions[item.path] = version + path = os.fsencode(os.path.normpath(item.path)) + self.file_versions[path] = version del item.path # safe some space if 'source' in item and stat.S_ISREG(item.mode): # a hardlink, no contents, is the hardlink master - source = item.source + source = os.fsencode(os.path.normpath(item.source)) if self.versions: # adjust source name with version version = self.file_versions[source] - source = os.fsdecode(make_versioned_name(os.fsencode(source), version, add_dir=True)) + source = make_versioned_name(source, version, add_dir=True) name = make_versioned_name(name, version) inode = self._find_inode(source, prefix) item = self.cache.get(inode) @@ -236,7 +238,7 @@ class FuseOperations(llfuse.Operations): return self.cache.get(inode) def _find_inode(self, path, prefix=[]): - segments = prefix + os.fsencode(os.path.normpath(path)).split(b'/') + segments = prefix + path.split(b'/') inode = 1 for segment in segments: inode = self.contents[inode][segment]