diff --git a/.travis.yml b/.travis.yml index 58b97c34..5e73b62e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,44 +9,40 @@ matrix: include: - python: "3.6" os: linux - dist: trusty - env: TOXENV=py36 + dist: bionic + env: TOXENV=py36-fuse2 - python: "3.7" os: linux - dist: xenial - env: TOXENV=py37 - - python: "3.7-dev" - os: linux - dist: xenial - env: TOXENV=py37 + dist: bionic + env: TOXENV=py37-fuse2 - python: "3.8" os: linux - dist: xenial - env: TOXENV=py38 + dist: focal + env: TOXENV=py38-fuse2 - python: "3.8-dev" os: linux - dist: xenial - env: TOXENV=py38 - - python: "3.9-dev" - os: linux - dist: xenial - env: TOXENV=py39 + dist: focal + env: TOXENV=py38-fuse3 - python: "3.9-dev" os: linux dist: focal - env: TOXENV=py39 - - python: "3.6" + env: TOXENV=py39-fuse2 + - python: "3.9-dev" os: linux - dist: xenial + dist: focal + env: TOXENV=py39-fuse3 + - python: "3.8" + os: linux + dist: focal env: TOXENV=flake8 - language: generic os: osx osx_image: xcode8.3 # This is the latest working xcode image with osxfuse compatibility; later images come with an OS X version which doesn't allow kernel extensions - env: TOXENV=py36 + env: TOXENV=py36-fuse2 - language: generic os: osx osx_image: xcode11.3 - env: TOXENV=py37 SKIPFUSE=true + env: TOXENV=py37 # No FUSE testing, because recent versions of macOS don't allow kernel extensions of osxfuse. allow_failures: - os: osx # OS X builds often take too long and time out, even though tests don't actually fail diff --git a/.travis/install.sh b/.travis/install.sh index 8c2e6781..77bf7f09 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -48,7 +48,8 @@ then #sudo apt-get install -y liblz4-dev # Too old on trusty and xenial, but might be useful in future versions #sudo apt-get install -y libzstd-dev # Too old on trusty and xenial, but might be useful in future versions sudo apt-get install -y libacl1-dev - sudo apt-get install -y libfuse-dev fuse # Required for Python llfuse module + sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module + sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module else @@ -67,12 +68,3 @@ pip install -r requirements.d/development.txt pip install codecov python setup.py --version -# Recent versions of OS X don't allow kernel extensions which makes the osxfuse tests fail; those versions are marked with SKIPFUSE=true in .travis.yml -if [ "${SKIPFUSE}" = "true" ] -then - truncate -s 0 requirements.d/fuse.txt - pip install -e . -else - pip install -e .[fuse] -fi - diff --git a/Vagrantfile b/Vagrantfile index e99b8a42..2266d590 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,9 @@ def packages_debianoid(user) apt-get -y -qq update apt-get -y -qq dist-upgrade # for building borgbackup and dependencies: - apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev libfuse-dev fuse pkg-config + apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config + apt install -y libfuse-dev fuse || true + apt install -y libfuse3-dev fuse3 || true usermod -a -G fuse #{user} chgrp fuse /dev/fuse chmod 666 /dev/fuse @@ -43,7 +45,9 @@ def packages_freebsd # install all the (security and other) updates, base system freebsd-update --not-running-from-cron fetch install # for building borgbackup and dependencies: - pkg install -y liblz4 zstd fusefs-libs pkgconf + pkg install -y liblz4 zstd pkgconf + pkg install -y fusefs-libs || true + pkg install -y fusefs-libs3 || true pkg install -y git bash # fakeroot causes lots of troubles on freebsd # for building python: pkg install -y python37 py37-sqlite3 py37-virtualenv py37-pip @@ -160,7 +164,7 @@ def build_pyenv_venv(boxname) end def install_borg(fuse) - script = <<-EOF + return <<-EOF . ~/.bash_profile cd /vagrant/borg . borg-env/bin/activate @@ -168,20 +172,8 @@ def install_borg(fuse) cd borg pip install -r requirements.d/development.txt python setup.py clean + pip install -e .[#{fuse}] EOF - if fuse - script += <<-EOF - # by using [fuse], setup.py can handle different FUSE requirements: - pip install -e .[fuse] - EOF - else - script += <<-EOF - pip install -e . - # do not install llfuse into the virtualenvs built by tox: - sed -i.bak '/fuse.txt/d' tox.ini - EOF - end - return script end def install_pyinstaller() @@ -221,10 +213,10 @@ def run_tests(boxname) # otherwise: just use the system python if which fakeroot 2> /dev/null; then echo "Running tox WITH fakeroot -u" - fakeroot -u tox --skip-missing-interpreters -e py36,py37,py38,py39 + fakeroot -u tox --skip-missing-interpreters else echo "Running tox WITHOUT fakeroot -u" - tox --skip-missing-interpreters -e py36,py37,py38,py39 + tox --skip-missing-interpreters fi EOF end @@ -263,7 +255,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("focal64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("focal64") end @@ -275,7 +267,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("bionic64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bionic64") end @@ -289,7 +281,7 @@ Vagrant.configure(2) do |config| b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("buster64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("buster64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("buster64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("buster64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64") @@ -305,7 +297,7 @@ Vagrant.configure(2) do |config| b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64") @@ -319,7 +311,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages arch", :type => :shell, :privileged => true, :inline => packages_arch b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("arch64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("arch64") end @@ -334,7 +326,7 @@ Vagrant.configure(2) do |config| b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd64") @@ -349,7 +341,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") end @@ -373,7 +365,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64") b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse") b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64") @@ -389,7 +381,7 @@ Vagrant.configure(2) do |config| b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant") b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64") end diff --git a/conftest.py b/conftest.py index 6fd59610..d1505033 100644 --- a/conftest.py +++ b/conftest.py @@ -21,7 +21,7 @@ from borg.logger import setup_logging # Ensure that the loggers exist for all tests setup_logging() -from borg.testsuite import has_lchflags, has_llfuse +from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3 from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported from borg.testsuite.platform import fakeroot_detected, are_acls_working from borg import xattr @@ -33,7 +33,8 @@ def clean_env(tmpdir_factory, monkeypatch): monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home'))) monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home'))) # also avoid to use anything from the outside environment: - keys = [key for key in os.environ if key.startswith('BORG_')] + keys = [key for key in os.environ + if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )] for key in keys: monkeypatch.delenv(key, raising=False) @@ -41,7 +42,8 @@ def clean_env(tmpdir_factory, monkeypatch): def pytest_report_header(config, startdir): tests = { "BSD flags": has_lchflags, - "fuse": has_llfuse, + "fuse2": has_llfuse, + "fuse3": has_pyfuse3, "root": not fakeroot_detected(), "symlinks": are_symlinks_supported(), "hardlinks": are_hardlinks_supported(), diff --git a/docs/development.rst b/docs/development.rst index 7e31131c..5abe3acd 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -288,7 +288,7 @@ Usage:: Creating standalone binaries ---------------------------- -Make sure you have everything built and installed (including llfuse and fuse). +Make sure you have everything built and installed (including fuse stuff). When using the Vagrant VMs, pyinstaller will already be installed. With virtual env activated:: diff --git a/docs/global.rst.inc b/docs/global.rst.inc index ead84340..14725f73 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -22,6 +22,7 @@ .. _msgpack: https://msgpack.org/ .. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/ .. _llfuse: https://pypi.python.org/pypi/llfuse/ +.. _pyfuse3: https://pypi.python.org/pypi/pyfuse3/ .. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace .. _Cython: http://cython.org/ .. _virtualenv: https://pypi.python.org/pypi/virtualenv/ diff --git a/docs/installation.rst b/docs/installation.rst index 860b6993..c7711cbc 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -159,8 +159,12 @@ following dependencies first: it will fall back to using the bundled code, see above). These must be present before invoking setup.py! * some other Python dependencies, pip will automatically install them for you. -* optionally, the llfuse_ Python package is required if you wish to mount an - archive as a FUSE filesystem. See setup.py about the version requirements. +* optionally, if you wish to mount an archive as a FUSE filesystem, you need + a FUSE implementation for Python: + + - Either pyfuse3_ (preferably, newer and maintained) or llfuse_ (older, + unmaintained now). See also the BORG_FUSE_IMPL env variable. + - See setup.py about the version requirements. If you have troubles finding the right package names, have a look at the distribution specific sections below or the Vagrantfile in the git repository, @@ -186,7 +190,8 @@ Install the dependencies with development headers:: liblz4-dev libzstd-dev \ build-essential \ pkg-config python3-pkgconfig - sudo apt-get install libfuse-dev fuse # optional, for FUSE support + sudo apt-get install libfuse-dev fuse # needed for llfuse + sudo apt-get install libfuse3-dev fuse3 # needed for pyfuse3 In case you get complaints about permission denied on ``/etc/fuse.conf``: on Ubuntu this means your user is not in the ``fuse`` group. Add yourself to that @@ -203,7 +208,8 @@ Install the dependencies with development headers:: lz4-devel libzstd-devel \ pkgconf python3-pkgconfig sudo dnf install gcc gcc-c++ redhat-rpm-config - sudo dnf install fuse-devel fuse # optional, for FUSE support + sudo dnf install fuse-devel fuse # needed for llfuse + sudo dnf install fuse3-devel fuse3 # needed for pyfuse3 openSUSE Tumbleweed / Leap ++++++++++++++++++++++++++ @@ -218,7 +224,8 @@ Alternatively, you can enumerate all build dependencies in the command line:: libacl-devel openssl-devel \ python3-Cython python3-Sphinx python3-msgpack-python \ python3-pytest python3-setuptools python3-setuptools_scm \ - python3-sphinx_rtd_theme python3-llfuse gcc gcc-c++ + python3-sphinx_rtd_theme gcc gcc-c++ + sudo zypper install python3-llfuse # llfuse Mac OS X ++++++++ @@ -234,7 +241,7 @@ For FUSE support to mount the backup archives, you need at least version 3.0 of FUSE for OS X, which is available via `github `__, or via Homebrew:: - brew cask install osxfuse + brew cask install osxfuse # needed for llfuse FreeBSD @@ -248,7 +255,7 @@ and commands to make FUSE work for using the mount command. pkg install -y python3 pkgconf pkg install openssl pkg install liblz4 zstd - pkg install fusefs-libs + pkg install fusefs-libs # needed for llfuse pkg install -y git python3.5 -m ensurepip # to install pip for Python3 To use the mount command: @@ -308,15 +315,17 @@ This will use ``pip`` to install the latest release from PyPi:: # might be required if your tools are outdated pip install -U pip setuptools wheel + # install Borg + Python dependencies into virtualenv pip install borgbackup # or alternatively (if you want FUSE support): - pip install borgbackup[fuse] + pip install borgbackup[llfuse] # to use llfuse + pip install borgbackup[pyfuse3] # to use pyfuse3 To upgrade Borg to a new version later, run the following after activating your virtual environment:: - pip install -U borgbackup # or ... borgbackup[fuse] + pip install -U borgbackup # or ... borgbackup[llfuse/pyfuse3] .. _git-installation: @@ -339,8 +348,12 @@ While we try not to break master, there are no guarantees on anything. cd borg pip install -r requirements.d/development.txt pip install -r requirements.d/docs.txt # optional, to build the docs - pip install -r requirements.d/fuse.txt # optional, for FUSE support - pip install -e . # in-place editable mode + + pip install -e . # in-place editable mode + or + pip install -e .[pyfuse3] # in-place editable mode, use pyfuse3 + or + pip install -e .[llfuse] # in-place editable mode, use llfuse # optional: run all the tests, on all supported Python versions # requires fakeroot, available through your package manager diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index 87a3f84b..ae796be3 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -64,6 +64,16 @@ General: When set to no (default: yes), system information (like OS, Python version, ...) in exceptions is not shown. Please only use for good reasons as it makes issues harder to analyze. + BORG_FUSE_IMPL + Choose the lowlevel FUSE implementation borg shall use for ``borg mount``. + This is a comma-separated list of implementation names, they are tried in the + given order, e.g.: + + - ``pyfuse3,llfuse``: default, first try to load pyfuse3, then try to load llfuse. + - ``llfuse,pyfuse3``: first try to load llfuse, then try to load pyfuse3. + - ``pyfuse3``: only try to load pyfuse3 + - ``llfuse``: only try to load llfuse + - ``none``: do not try to load an implementation BORG_WORKAROUNDS A list of comma separated strings that trigger workarounds in borg, e.g. to work around bugs in other software. diff --git a/requirements.d/fuse.txt b/requirements.d/fuse.txt deleted file mode 100644 index 08dee458..00000000 --- a/requirements.d/fuse.txt +++ /dev/null @@ -1,4 +0,0 @@ -# low-level FUSE support library for "borg mount" -# please see the comments in setup.py about llfuse. -llfuse >=1.3.4, <1.3.7; python_version <"3.9" # broken on py39 -llfuse >=1.3.7, <2.0; python_version >="3.9" # broken on freebsd diff --git a/setup.py b/setup.py index e19159c8..4152f9da 100644 --- a/setup.py +++ b/setup.py @@ -78,14 +78,17 @@ install_requires = [ ] # note for package maintainers: if you package borgbackup for distribution, -# please add llfuse as a *requirement* on all platforms that have a working -# llfuse package. "borg mount" needs llfuse to work. -# if you do not have llfuse, do not require it, most of borgbackup will work. +# please (if available) add pyfuse3 (preferably) or llfuse (not maintained any more) +# as a *requirement*. "borg mount" needs one of them to work. +# if neither is available, do not require it, most of borgbackup will work. extras_require = { - 'fuse': [ - 'llfuse >=1.3.4, <1.3.7; python_version <"3.9"', # broken on py39 - 'llfuse >=1.3.7, <2.0; python_version >="3.9"', # broken on freebsd + 'llfuse': [ + 'llfuse >= 1.3.8', ], + 'pyfuse3': [ + 'pyfuse3 >= 3.1.1', + ], + 'nofuse': [], } compress_source = 'src/borg/compress.pyx' diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 53832609..02bfe1ad 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1258,10 +1258,9 @@ class Archiver: """Mount archive or an entire repository as a FUSE filesystem""" # Perform these checks before opening the repository and asking for a passphrase. - try: - import borg.fuse - except ImportError as e: - self.print_error('borg mount not available: loading FUSE support failed [ImportError: %s]' % str(e)) + from .fuse_impl import llfuse, BORG_FUSE_IMPL + if llfuse is None: + self.print_error('borg mount not available: no FUSE support, BORG_FUSE_IMPL=%s.' % BORG_FUSE_IMPL) return self.exit_code if not os.path.isdir(args.mountpoint) or not os.access(args.mountpoint, os.R_OK | os.W_OK | os.X_OK): diff --git a/src/borg/fuse.py b/src/borg/fuse.py index bc8b59be..841a7261 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -1,4 +1,5 @@ import errno +import functools import io import os import stat @@ -9,7 +10,23 @@ import time from collections import defaultdict from signal import SIGINT -import llfuse +from .fuse_impl import llfuse, has_pyfuse3 + + +if has_pyfuse3: + import trio + + def async_wrapper(fn): + @functools.wraps(fn) + async def wrapper(*args, **kwargs): + return fn(*args, **kwargs) + return wrapper +else: + trio = None + + def async_wrapper(fn): + return fn + from .logger import create_logger logger = create_logger() @@ -26,7 +43,15 @@ from .remote import RemoteRepository def fuse_main(): - return llfuse.main(workers=1) + if has_pyfuse3: + try: + trio.run(llfuse.main) + except: + return 1 # TODO return signal number if it was killed by signal + else: + return None + else: + return llfuse.main(workers=1) # size of some LRUCaches (1 element per simultaneously open file) @@ -533,6 +558,7 @@ class FuseOperations(llfuse.Operations, FuseBackend): finally: llfuse.close(umount) + @async_wrapper def statfs(self, ctx=None): stat_ = llfuse.StatvfsData() stat_.f_bsize = 512 @@ -546,7 +572,7 @@ class FuseOperations(llfuse.Operations, FuseBackend): stat_.f_namemax = 255 # == NAME_MAX (depends on archive source OS / FS) return stat_ - def getattr(self, inode, ctx=None): + def _getattr(self, inode, ctx=None): item = self.get_item(inode) entry = llfuse.EntryAttributes() entry.st_ino = inode @@ -568,10 +594,16 @@ class FuseOperations(llfuse.Operations, FuseBackend): entry.st_birthtime_ns = item.get('birthtime', mtime_ns) return entry + @async_wrapper + def getattr(self, inode, ctx=None): + return self._getattr(inode, ctx=ctx) + + @async_wrapper def listxattr(self, inode, ctx=None): item = self.get_item(inode) return item.get('xattrs', {}).keys() + @async_wrapper def getxattr(self, inode, name, ctx=None): item = self.get_item(inode) try: @@ -579,6 +611,7 @@ class FuseOperations(llfuse.Operations, FuseBackend): except KeyError: raise llfuse.FUSEError(llfuse.ENOATTR) from None + @async_wrapper def lookup(self, parent_inode, name, ctx=None): self.check_pending_archive(parent_inode) if name == b'.': @@ -589,8 +622,9 @@ class FuseOperations(llfuse.Operations, FuseBackend): inode = self.contents[parent_inode].get(name) if not inode: raise llfuse.FUSEError(errno.ENOENT) - return self.getattr(inode) + return self._getattr(inode) + @async_wrapper def open(self, inode, flags, ctx=None): if not self.allow_damaged_files: item = self.get_item(inode) @@ -601,12 +635,14 @@ class FuseOperations(llfuse.Operations, FuseBackend): logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. ' 'Mount with allow_damaged_files to read damaged files.') raise llfuse.FUSEError(errno.EIO) - return inode + return llfuse.FileInfo(fh=inode) if has_pyfuse3 else inode + @async_wrapper def opendir(self, inode, ctx=None): self.check_pending_archive(inode) return inode + @async_wrapper def read(self, fh, offset, size): parts = [] item = self.get_item(fh) @@ -650,12 +686,25 @@ class FuseOperations(llfuse.Operations, FuseBackend): break return b''.join(parts) - def readdir(self, fh, off): - entries = [(b'.', fh), (b'..', self.parent[fh])] - entries.extend(self.contents[fh].items()) - for i, (name, inode) in enumerate(entries[off:], off): - yield name, self.getattr(inode), i + 1 + # note: we can't have a generator (with yield) and not a generator (async) in the same method + if has_pyfuse3: + async def readdir(self, fh, off, token): + entries = [(b'.', fh), (b'..', self.parent[fh])] + entries.extend(self.contents[fh].items()) + for i, (name, inode) in enumerate(entries[off:], off): + attrs = self._getattr(inode) + if not llfuse.readdir_reply(token, name, attrs, i + 1): + break + else: + def readdir(self, fh, off): + entries = [(b'.', fh), (b'..', self.parent[fh])] + entries.extend(self.contents[fh].items()) + for i, (name, inode) in enumerate(entries[off:], off): + attrs = self._getattr(inode) + yield name, attrs, i + 1 + + @async_wrapper def readlink(self, inode, ctx=None): item = self.get_item(inode) return os.fsencode(item.source) diff --git a/src/borg/fuse_impl.py b/src/borg/fuse_impl.py new file mode 100644 index 00000000..390ac576 --- /dev/null +++ b/src/borg/fuse_impl.py @@ -0,0 +1,36 @@ +""" +load library for lowlevel FUSE implementation +""" + +import os + +BORG_FUSE_IMPL = os.environ.get('BORG_FUSE_IMPL', 'pyfuse3,llfuse') + +for FUSE_IMPL in BORG_FUSE_IMPL.split(','): + FUSE_IMPL = FUSE_IMPL.strip() + if FUSE_IMPL == 'pyfuse3': + try: + import pyfuse3 as llfuse + except ImportError: + pass + else: + has_llfuse = False + has_pyfuse3 = True + break + elif FUSE_IMPL == 'llfuse': + try: + import llfuse + except ImportError: + pass + else: + has_llfuse = True + has_pyfuse3 = False + break + elif FUSE_IMPL == 'none': + pass + else: + raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL) +else: + llfuse = None + has_llfuse = False + has_pyfuse3 = False diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 844db8a8..be4b7c04 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -23,12 +23,10 @@ from .. import platform # Note: this is used by borg.selftest, do not use or import py.test functionality here. -try: - import llfuse - # Does this version of llfuse support ns precision? - have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') -except ImportError: - have_fuse_mtime_ns = False +from ..fuse_impl import llfuse, has_pyfuse3, has_llfuse + +# Does this version of llfuse support ns precision? +have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False try: from pytest import raises @@ -42,12 +40,6 @@ try: except OSError: has_lchflags = False -try: - import llfuse - has_llfuse = True or llfuse # avoids "unused import" -except ImportError: - has_llfuse = False - # The mtime get/set precision varies on different OS and Python versions if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): st_mtime_ns_round = 0 diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index d749c8ab..1f3a2efd 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -26,11 +26,6 @@ from unittest.mock import patch import pytest -try: - import llfuse -except ImportError: - pass - import borg from .. import xattr, helpers, platform from ..archive import Archive, ChunkBuffer @@ -55,7 +50,7 @@ from ..locking import LockFailed from ..logger import setup_logging from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository -from . import has_lchflags, has_llfuse +from . import has_lchflags, llfuse from . import BaseTestCase, changedir, environment_variable, no_selinux from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported from .platform import fakeroot_detected @@ -798,7 +793,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') @requires_hardlinks - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_fuse_mount_hardlinks(self): self._extract_hardlinks_setup() mountpoint = os.path.join(self.tmpdir, 'mountpoint') @@ -1661,7 +1656,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): # verify that command works with read-only repo when using --bypass-lock self.cmd('list', self.repository_location, '--bypass-lock') - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_readonly_mount(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_src_archive('test') @@ -1754,7 +1749,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): # delete of the whole repository ignores features self.cmd('delete', self.repository_location) - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_unknown_feature_on_mount(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') @@ -2322,7 +2317,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert 'positional arguments' not in self.cmd('help', 'init', '--epilog-only') assert 'This command initializes' not in self.cmd('help', 'init', '--usage-only') - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_fuse(self): def has_noatime(some_file): atime_before = os.stat(some_file).st_atime_ns @@ -2423,7 +2418,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): else: raise - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_fuse_versions_view(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_regular_file('test', contents=b'first') @@ -2455,7 +2450,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert os.stat(hl2).st_ino == os.stat(hl3).st_ino assert open(hl3, 'rb').read() == b'123456' - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_src_archive('archive') @@ -2480,7 +2475,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): with self.fuse_mount(self.repository_location + '::archive', mountpoint, '-o', 'allow_damaged_files'): open(os.path.join(mountpoint, path)).close() - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_fuse_mount_options(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_src_archive('arch11') @@ -2502,7 +2497,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): with self.fuse_mount(self.repository_location, mountpoint, '--prefix=nope'): assert sorted(os.listdir(os.path.join(mountpoint))) == [] - @unittest.skipUnless(has_llfuse, 'llfuse not installed') + @unittest.skipUnless(llfuse, 'llfuse not installed') def test_migrate_lock_alive(self): """Both old_id and new_id must not be stale during lock migration / daemonization.""" from functools import wraps diff --git a/tox.ini b/tox.ini index f84cc26a..b2f597de 100644 --- a/tox.ini +++ b/tox.ini @@ -2,16 +2,29 @@ # fakeroot -u tox --recreate [tox] -envlist = py{36,37,38,39},flake8 +envlist = py{36,37,38,39}-fuse{2,3}, flake8 [testenv] deps = - -rrequirements.d/development.txt - -rrequirements.d/fuse.txt + -rrequirements.d/development.txt commands = py.test -v -n {env:XDISTN:1} -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = * +[testenv:py{36,37,38,39}-fuse2] +setenv = + BORG_FUSE_IMPL=llfuse +deps = + llfuse + {[testenv]deps} + +[testenv:py{36,37,38,39}-fuse3] +setenv = + BORG_FUSE_IMPL=pyfuse3 +deps = + pyfuse3 + {[testenv]deps} + [testenv:flake8] changedir = deps =