From cb4a52eb84fe05ad6276b0bd0c62f3c059ed0a9f Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 10 Jul 2016 02:19:27 +0200 Subject: [PATCH 1/3] fuse: EIO on damaged files unless told not to (-o allow_damaged_files) --- borg/archiver.py | 7 +++++++ borg/fuse.py | 25 +++++++++++++++++++++++-- borg/testsuite/__init__.py | 7 +++++-- borg/testsuite/archiver.py | 38 +++++++++++++++++++++++++++++++------- 4 files changed, 66 insertions(+), 11 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index ff1463708..c40f8c091 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -1267,6 +1267,13 @@ def build_parser(self, args=None, prog=None): browsing an archive or restoring individual files. Unless the ``--foreground`` option is given the command will run in the background until the filesystem is ``umounted``. + + For mount options, see the fuse(8) manual page. Additional mount options + supported by borg: + + - allow_damaged_files: by default damaged files (where missing chunks were + replaced with runs of zeros by borg check --repair) are not readable and + return EIO (I/O error). Set this option to read such files. """) subparser = subparsers.add_parser('mount', parents=[common_parser], description=self.do_mount.__doc__, diff --git a/borg/fuse.py b/borg/fuse.py index 19f8d71f2..b1387f200 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -6,11 +6,16 @@ import stat import tempfile import time -from .archive import Archive -from .helpers import daemonize, bigint_to_int from distutils.version import LooseVersion + import msgpack +from .archive import Archive +from .helpers import daemonize, bigint_to_int, remove_surrogates +from .logger import create_logger +logger = create_logger() + + # Does this version of llfuse support ns precision? have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') @@ -42,6 +47,8 @@ def get(self, inode): class FuseOperations(llfuse.Operations): """Export archive as a fuse filesystem """ + allow_damaged_files = True + def __init__(self, key, repository, manifest, archive, cached_repo): super().__init__() self._inode_count = 0 @@ -225,6 +232,15 @@ def lookup(self, parent_inode, name, ctx=None): return self.getattr(inode) def open(self, inode, flags, ctx=None): + if not self.allow_damaged_files: + item = self.get_item(inode) + if b'chunks_healthy' in item: + # Processed archive items don't carry the path anymore; for converting the inode + # to the path we'd either have to store the inverse of the current structure, + # or search the entire archive. So we just don't print it. It's easy to correlate anyway. + logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. ' + 'Mount with allow_damaged_files to read damaged files.') + raise llfuse.FUSEError(errno.EIO) return inode def opendir(self, inode, ctx=None): @@ -261,6 +277,11 @@ def mount(self, mountpoint, extra_options, foreground=False): options = ['fsname=borgfs', 'ro'] if extra_options: options.extend(extra_options.split(',')) + try: + options.remove('allow_damaged_files') + self.allow_damaged_files = True + except ValueError: + self.allow_damaged_files = False llfuse.init(self, mountpoint, options) if not foreground: daemonize() diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index b6f64b19c..64c240c2f 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -94,9 +94,12 @@ def _assert_dirs_equal_cmp(self, diff): self._assert_dirs_equal_cmp(sub_diff) @contextmanager - def fuse_mount(self, location, mountpoint): + def fuse_mount(self, location, mountpoint, mount_options=None): os.mkdir(mountpoint) - self.cmd('mount', location, mountpoint, fork=True) + args = ['mount', location, mountpoint] + if mount_options: + args += '-o', mount_options + self.cmd(*args, fork=True) self.wait_for_mount(mountpoint) yield if sys.platform.startswith('linux'): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d52391239..6d08248fa 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -234,6 +234,13 @@ def cmd(self, *args, **kw): def create_src_archive(self, name): self.cmd('create', self.repository_location + '::' + name, src_dir) + def open_archive(self, name): + repository = Repository(self.repository_path) + with repository: + manifest, key = Manifest.load(repository) + archive = Archive(repository, key, manifest, name) + return archive, repository + class ArchiverTestCase(ArchiverTestCaseBase): @@ -1037,6 +1044,30 @@ def test_fuse(self): sto = os.stat(out_fn) assert stat.S_ISFIFO(sto.st_mode) + @unittest.skipUnless(has_llfuse, 'llfuse not installed') + def test_fuse_allow_damaged_files(self): + self.cmd('init', self.repository_location) + self.create_src_archive('archive') + # Get rid of a chunk and repair it + archive, repository = self.open_archive('archive') + with repository: + for item in archive.iter_items(): + if item[b'path'].endswith('testsuite/archiver.py'): + repository.delete(item[b'chunks'][-1][0]) + path = item[b'path'] # store full path for later + break + else: + assert False # missed the file + repository.commit() + self.cmd('check', '--repair', self.repository_location, exit_code=0) + + mountpoint = os.path.join(self.tmpdir, 'mountpoint') + with self.fuse_mount(self.repository_location + '::archive', mountpoint): + with pytest.raises(OSError): + open(os.path.join(mountpoint, path)) + with self.fuse_mount(self.repository_location + '::archive', mountpoint, 'allow_damaged_files'): + open(os.path.join(mountpoint, path)).close() + def verify_aes_counter_uniqueness(self, method): seen = set() # Chunks already seen used = set() # counter values already used @@ -1117,13 +1148,6 @@ def setUp(self): self.create_src_archive('archive1') self.create_src_archive('archive2') - def open_archive(self, name): - repository = Repository(self.repository_path) - with repository: - manifest, key = Manifest.load(repository) - archive = Archive(repository, key, manifest, name) - return archive, repository - def test_check_usage(self): output = self.cmd('check', '-v', self.repository_location, exit_code=0) self.assert_in('Starting repository check', output) From 5a952ec6df256e65303552cdbe883f9d84999a16 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 10 Jul 2016 00:20:11 +0200 Subject: [PATCH 2/3] fuse: clean imports, move mount to after __init__ * * since the two are always called in direct succession and perform related setup. --- borg/fuse.py | 56 +++++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/borg/fuse.py b/borg/fuse.py index b1387f200..7b75b8b8e 100644 --- a/borg/fuse.py +++ b/borg/fuse.py @@ -11,7 +11,7 @@ import msgpack from .archive import Archive -from .helpers import daemonize, bigint_to_int, remove_surrogates +from .helpers import daemonize, bigint_to_int from .logger import create_logger logger = create_logger() @@ -47,7 +47,8 @@ def get(self, inode): class FuseOperations(llfuse.Operations): """Export archive as a fuse filesystem """ - allow_damaged_files = True + + allow_damaged_files = False def __init__(self, key, repository, manifest, archive, cached_repo): super().__init__() @@ -75,6 +76,32 @@ def __init__(self, key, repository, manifest, archive, cached_repo): self.contents[1][os.fsencode(archive_name)] = archive_inode self.pending_archives[archive_inode] = Archive(repository, key, manifest, archive_name) + def mount(self, mountpoint, mount_options, foreground=False): + """Mount filesystem on *mountpoint* with *mount_options*.""" + options = ['fsname=borgfs', 'ro'] + if mount_options: + options.extend(mount_options.split(',')) + try: + options.remove('allow_damaged_files') + self.allow_damaged_files = True + except ValueError: + pass + llfuse.init(self, mountpoint, options) + if not foreground: + daemonize() + + # If the file system crashes, we do not want to umount because in that + # case the mountpoint suddenly appears to become empty. This can have + # nasty consequences, imagine the user has e.g. an active rsync mirror + # job - seeing the mountpoint empty, rsync would delete everything in the + # mirror. + umount = False + try: + signal = fuse_main() + umount = (signal is None) # no crash and no signal -> umount request + finally: + llfuse.close(umount) + def process_archive(self, archive, prefix=[]): """Build fuse inode hierarchy from archive metadata """ @@ -272,28 +299,3 @@ def readdir(self, fh, off): def readlink(self, inode, ctx=None): item = self.get_item(inode) return os.fsencode(item[b'source']) - - def mount(self, mountpoint, extra_options, foreground=False): - options = ['fsname=borgfs', 'ro'] - if extra_options: - options.extend(extra_options.split(',')) - try: - options.remove('allow_damaged_files') - self.allow_damaged_files = True - except ValueError: - self.allow_damaged_files = False - llfuse.init(self, mountpoint, options) - if not foreground: - daemonize() - - # If the file system crashes, we do not want to umount because in that - # case the mountpoint suddenly appears to become empty. This can have - # nasty consequences, imagine the user has e.g. an active rsync mirror - # job - seeing the mountpoint empty, rsync would delete everything in the - # mirror. - umount = False - try: - signal = fuse_main() - umount = (signal is None) # no crash and no signal -> umount request - finally: - llfuse.close(umount) From 1c7bdcdf9c925d5c6fa89d07cc6b1fc3fdc4dc1b Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 10 Jul 2016 17:25:38 +0200 Subject: [PATCH 3/3] test_fuse_allow_damaged_files: assert exact error code --- borg/testsuite/archiver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 6d08248fa..3e382eda8 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1063,8 +1063,9 @@ def test_fuse_allow_damaged_files(self): mountpoint = os.path.join(self.tmpdir, 'mountpoint') with self.fuse_mount(self.repository_location + '::archive', mountpoint): - with pytest.raises(OSError): + with pytest.raises(OSError) as excinfo: open(os.path.join(mountpoint, path)) + assert excinfo.value.errno == errno.EIO with self.fuse_mount(self.repository_location + '::archive', mountpoint, 'allow_damaged_files'): open(os.path.join(mountpoint, path)).close()