mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

FUSE: support pyfuse3 additionally to llfuse, fixes #5407 

FUSE implementation can be switched via env var BORG_FUSE_IMPL.
This commit is contained in:
Thomas Waldmann 2020-10-10 23:12:47 +02:00
parent d56c816cf2
commit 49b1421682
16 changed files with 215 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
.travis.yml
View File

@ -9,44 +9,40 @@ matrix:
include:
- python: "3.6"
os: linux
dist: trusty
env: TOXENV=py36
dist: bionic
env: TOXENV=py36-fuse2
- python: "3.7"
os: linux
dist: xenial
env: TOXENV=py37
- python: "3.7-dev"
os: linux
dist: xenial
env: TOXENV=py37
dist: bionic
env: TOXENV=py37-fuse2
- python: "3.8"
os: linux
dist: xenial
env: TOXENV=py38
dist: focal
env: TOXENV=py38-fuse2
- python: "3.8-dev"
os: linux
dist: xenial
env: TOXENV=py38
- python: "3.9-dev"
os: linux
dist: xenial
env: TOXENV=py39
dist: focal
env: TOXENV=py38-fuse3
- python: "3.9-dev"
os: linux
dist: focal
env: TOXENV=py39
- python: "3.6"
env: TOXENV=py39-fuse2
- python: "3.9-dev"
os: linux
dist: xenial
dist: focal
env: TOXENV=py39-fuse3
- python: "3.8"
os: linux
dist: focal
env: TOXENV=flake8
- language: generic
os: osx
osx_image: xcode8.3 # This is the latest working xcode image with osxfuse compatibility; later images come with an OS X version which doesn't allow kernel extensions
env: TOXENV=py36
env: TOXENV=py36-fuse2
- language: generic
os: osx
osx_image: xcode11.3
env: TOXENV=py37 SKIPFUSE=true
env: TOXENV=py37 # No FUSE testing, because recent versions of macOS don't allow kernel extensions of osxfuse.
allow_failures:
- os: osx # OS X builds often take too long and time out, even though tests don't actually fail

12
.travis/install.sh
View File

@ -48,7 +48,8 @@ then
#sudo apt-get install -y liblz4-dev # Too old on trusty and xenial, but might be useful in future versions
#sudo apt-get install -y libzstd-dev # Too old on trusty and xenial, but might be useful in future versions
sudo apt-get install -y libacl1-dev
sudo apt-get install -y libfuse-dev fuse # Required for Python llfuse module
sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module
sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module
else
@ -67,12 +68,3 @@ pip install -r requirements.d/development.txt
pip install codecov
python setup.py --version
# Recent versions of OS X don't allow kernel extensions which makes the osxfuse tests fail; those versions are marked with SKIPFUSE=true in .travis.yml
if [ "${SKIPFUSE}" = "true" ]
then
truncate -s 0 requirements.d/fuse.txt
pip install -e .
else
pip install -e .[fuse]
fi

46
Vagrantfile vendored
View File

@ -15,7 +15,9 @@ def packages_debianoid(user)
apt-get -y -qq update
apt-get -y -qq dist-upgrade
# for building borgbackup and dependencies:
apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev libfuse-dev fuse pkg-config
apt install -y libssl-dev libacl1-dev liblz4-dev libzstd-dev pkg-config
apt install -y libfuse-dev fuse || true
apt install -y libfuse3-dev fuse3 || true
usermod -a -G fuse #{user}
chgrp fuse /dev/fuse
chmod 666 /dev/fuse
@ -43,7 +45,9 @@ def packages_freebsd
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y liblz4 zstd fusefs-libs pkgconf
pkg install -y liblz4 zstd pkgconf
pkg install -y fusefs-libs || true
pkg install -y fusefs-libs3 || true
pkg install -y git bash # fakeroot causes lots of troubles on freebsd
# for building python:
pkg install -y python37 py37-sqlite3 py37-virtualenv py37-pip
@ -160,7 +164,7 @@ def build_pyenv_venv(boxname)
end
def install_borg(fuse)
script = <<-EOF
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
@ -168,20 +172,8 @@ def install_borg(fuse)
cd borg
pip install -r requirements.d/development.txt
python setup.py clean
pip install -e .[#{fuse}]
EOF
if fuse
script += <<-EOF
# by using [fuse], setup.py can handle different FUSE requirements:
pip install -e .[fuse]
EOF
else
script += <<-EOF
pip install -e .
# do not install llfuse into the virtualenvs built by tox:
sed -i.bak '/fuse.txt/d' tox.ini
EOF
end
return script
end
def install_pyinstaller()
@ -221,10 +213,10 @@ def run_tests(boxname)
# otherwise: just use the system python
if which fakeroot 2> /dev/null; then
echo "Running tox WITH fakeroot -u"
fakeroot -u tox --skip-missing-interpreters -e py36,py37,py38,py39
fakeroot -u tox --skip-missing-interpreters
else
echo "Running tox WITHOUT fakeroot -u"
tox --skip-missing-interpreters -e py36,py37,py38,py39
tox --skip-missing-interpreters
fi
EOF
end
@ -263,7 +255,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("focal64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("focal64")
end
@ -275,7 +267,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("bionic64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bionic64")
end
@ -289,7 +281,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("buster64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("buster64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("buster64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("buster64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64")
@ -305,7 +297,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("stretch64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("stretch64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("stretch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64")
@ -319,7 +311,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages arch", :type => :shell, :privileged => true, :inline => packages_arch
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("arch64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("arch64")
end
@ -334,7 +326,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd64")
@ -349,7 +341,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
end
@ -373,7 +365,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64")
b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64")
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64")
@ -389,7 +381,7 @@ Vagrant.configure(2) do |config|
b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false)
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("nofuse")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64")
end

8
conftest.py
View File

@ -21,7 +21,7 @@ from borg.logger import setup_logging
# Ensure that the loggers exist for all tests
setup_logging()
from borg.testsuite import has_lchflags, has_llfuse
from borg.testsuite import has_lchflags, has_llfuse, has_pyfuse3
from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported
from borg.testsuite.platform import fakeroot_detected, are_acls_working
from borg import xattr
@ -33,7 +33,8 @@ def clean_env(tmpdir_factory, monkeypatch):
monkeypatch.setenv('XDG_CONFIG_HOME', str(tmpdir_factory.mktemp('xdg-config-home')))
monkeypatch.setenv('XDG_CACHE_HOME', str(tmpdir_factory.mktemp('xdg-cache-home')))
# also avoid to use anything from the outside environment:
keys = [key for key in os.environ if key.startswith('BORG_')]
keys = [key for key in os.environ
if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )]
for key in keys:
monkeypatch.delenv(key, raising=False)
@ -41,7 +42,8 @@ def clean_env(tmpdir_factory, monkeypatch):
def pytest_report_header(config, startdir):
tests = {
"BSD flags": has_lchflags,
"fuse": has_llfuse,
"fuse2": has_llfuse,
"fuse3": has_pyfuse3,
"root": not fakeroot_detected(),
"symlinks": are_symlinks_supported(),
"hardlinks": are_hardlinks_supported(),

2
docs/development.rst
View File

@ -288,7 +288,7 @@ Usage::
Creating standalone binaries
----------------------------
Make sure you have everything built and installed (including llfuse and fuse).
Make sure you have everything built and installed (including fuse stuff).
When using the Vagrant VMs, pyinstaller will already be installed.
With virtual env activated::

1
docs/global.rst.inc
View File

@ -22,6 +22,7 @@
.. _msgpack: https://msgpack.org/
.. _`msgpack-python`: https://pypi.python.org/pypi/msgpack-python/
.. _llfuse: https://pypi.python.org/pypi/llfuse/
.. _pyfuse3: https://pypi.python.org/pypi/pyfuse3/
.. _userspace filesystems: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
.. _Cython: http://cython.org/
.. _virtualenv: https://pypi.python.org/pypi/virtualenv/

35
docs/installation.rst
View File

@ -159,8 +159,12 @@ following dependencies first:
it will fall back to using the bundled code, see above).
These must be present before invoking setup.py!
* some other Python dependencies, pip will automatically install them for you.
* optionally, the llfuse_ Python package is required if you wish to mount an
archive as a FUSE filesystem. See setup.py about the version requirements.
* optionally, if you wish to mount an archive as a FUSE filesystem, you need
a FUSE implementation for Python:
- Either pyfuse3_ (preferably, newer and maintained) or llfuse_ (older,
unmaintained now). See also the BORG_FUSE_IMPL env variable.
- See setup.py about the version requirements.
If you have troubles finding the right package names, have a look at the
distribution specific sections below or the Vagrantfile in the git repository,
@ -186,7 +190,8 @@ Install the dependencies with development headers::
liblz4-dev libzstd-dev \
build-essential \
pkg-config python3-pkgconfig
sudo apt-get install libfuse-dev fuse # optional, for FUSE support
sudo apt-get install libfuse-dev fuse # needed for llfuse
sudo apt-get install libfuse3-dev fuse3 # needed for pyfuse3
In case you get complaints about permission denied on ``/etc/fuse.conf``: on
Ubuntu this means your user is not in the ``fuse`` group. Add yourself to that
@ -203,7 +208,8 @@ Install the dependencies with development headers::
lz4-devel libzstd-devel \
pkgconf python3-pkgconfig
sudo dnf install gcc gcc-c++ redhat-rpm-config
sudo dnf install fuse-devel fuse # optional, for FUSE support
sudo dnf install fuse-devel fuse # needed for llfuse
sudo dnf install fuse3-devel fuse3 # needed for pyfuse3
openSUSE Tumbleweed / Leap
++++++++++++++++++++++++++
@ -218,7 +224,8 @@ Alternatively, you can enumerate all build dependencies in the command line::
libacl-devel openssl-devel \
python3-Cython python3-Sphinx python3-msgpack-python \
python3-pytest python3-setuptools python3-setuptools_scm \
python3-sphinx_rtd_theme python3-llfuse gcc gcc-c++
python3-sphinx_rtd_theme gcc gcc-c++
sudo zypper install python3-llfuse # llfuse
Mac OS X
++++++++
@ -234,7 +241,7 @@ For FUSE support to mount the backup archives, you need at least version 3.0 of
FUSE for OS X, which is available via `github
<https://github.com/osxfuse/osxfuse/releases/latest>`__, or via Homebrew::
brew cask install osxfuse
brew cask install osxfuse # needed for llfuse
FreeBSD
@ -248,7 +255,7 @@ and commands to make FUSE work for using the mount command.
pkg install -y python3 pkgconf
pkg install openssl
pkg install liblz4 zstd
pkg install fusefs-libs
pkg install fusefs-libs # needed for llfuse
pkg install -y git
python3.5 -m ensurepip # to install pip for Python3
To use the mount command:
@ -308,15 +315,17 @@ This will use ``pip`` to install the latest release from PyPi::
# might be required if your tools are outdated
pip install -U pip setuptools wheel
# install Borg + Python dependencies into virtualenv
pip install borgbackup
# or alternatively (if you want FUSE support):
pip install borgbackup[fuse]
pip install borgbackup[llfuse] # to use llfuse
pip install borgbackup[pyfuse3] # to use pyfuse3
To upgrade Borg to a new version later, run the following after
activating your virtual environment::
pip install -U borgbackup # or ... borgbackup[fuse]
pip install -U borgbackup # or ... borgbackup[llfuse/pyfuse3]
.. _git-installation:
@ -339,8 +348,12 @@ While we try not to break master, there are no guarantees on anything.
cd borg
pip install -r requirements.d/development.txt
pip install -r requirements.d/docs.txt # optional, to build the docs
pip install -r requirements.d/fuse.txt # optional, for FUSE support
pip install -e . # in-place editable mode
pip install -e . # in-place editable mode
or
pip install -e .[pyfuse3] # in-place editable mode, use pyfuse3
or
pip install -e .[llfuse] # in-place editable mode, use llfuse
# optional: run all the tests, on all supported Python versions
# requires fakeroot, available through your package manager

10
docs/usage/general/environment.rst.inc
View File

@ -64,6 +64,16 @@ General:
When set to no (default: yes), system information (like OS, Python version, ...) in
exceptions is not shown.
Please only use for good reasons as it makes issues harder to analyze.
BORG_FUSE_IMPL
Choose the lowlevel FUSE implementation borg shall use for ``borg mount``.
This is a comma-separated list of implementation names, they are tried in the
given order, e.g.:
- ``pyfuse3,llfuse``: default, first try to load pyfuse3, then try to load llfuse.
- ``llfuse,pyfuse3``: first try to load llfuse, then try to load pyfuse3.
- ``pyfuse3``: only try to load pyfuse3
- ``llfuse``: only try to load llfuse
- ``none``: do not try to load an implementation
BORG_WORKAROUNDS
A list of comma separated strings that trigger workarounds in borg,
e.g. to work around bugs in other software.

4
requirements.d/fuse.txt
View File

@ -1,4 +0,0 @@
# low-level FUSE support library for "borg mount"
# please see the comments in setup.py about llfuse.
llfuse >=1.3.4, <1.3.7; python_version <"3.9" # broken on py39
llfuse >=1.3.7, <2.0; python_version >="3.9" # broken on freebsd

15
setup.py
View File

@ -78,14 +78,17 @@ install_requires = [
]
# note for package maintainers: if you package borgbackup for distribution,
# please add llfuse as a *requirement* on all platforms that have a working
# llfuse package. "borg mount" needs llfuse to work.
# if you do not have llfuse, do not require it, most of borgbackup will work.
# please (if available) add pyfuse3 (preferably) or llfuse (not maintained any more)
# as a *requirement*. "borg mount" needs one of them to work.
# if neither is available, do not require it, most of borgbackup will work.
extras_require = {
'fuse': [
'llfuse >=1.3.4, <1.3.7; python_version <"3.9"', # broken on py39
'llfuse >=1.3.7, <2.0; python_version >="3.9"', # broken on freebsd
'llfuse': [
'llfuse >= 1.3.8',
],
'pyfuse3': [
'pyfuse3 >= 3.1.1',
],
'nofuse': [],
}
compress_source = 'src/borg/compress.pyx'

7
src/borg/archiver.py
View File

@ -1258,10 +1258,9 @@ class Archiver:
"""Mount archive or an entire repository as a FUSE filesystem"""
# Perform these checks before opening the repository and asking for a passphrase.
try:
import borg.fuse
except ImportError as e:
self.print_error('borg mount not available: loading FUSE support failed [ImportError: %s]' % str(e))
from .fuse_impl import llfuse, BORG_FUSE_IMPL
if llfuse is None:
self.print_error('borg mount not available: no FUSE support, BORG_FUSE_IMPL=%s.' % BORG_FUSE_IMPL)
return self.exit_code
if not os.path.isdir(args.mountpoint) or not os.access(args.mountpoint, os.R_OK | os.W_OK | os.X_OK):

69
src/borg/fuse.py
View File

@ -1,4 +1,5 @@
import errno
import functools
import io
import os
import stat
@ -9,7 +10,23 @@ import time
from collections import defaultdict
from signal import SIGINT
import llfuse
from .fuse_impl import llfuse, has_pyfuse3
if has_pyfuse3:
import trio
def async_wrapper(fn):
@functools.wraps(fn)
async def wrapper(*args, **kwargs):
return fn(*args, **kwargs)
return wrapper
else:
trio = None
def async_wrapper(fn):
return fn
from .logger import create_logger
logger = create_logger()
@ -26,7 +43,15 @@ from .remote import RemoteRepository
def fuse_main():
return llfuse.main(workers=1)
if has_pyfuse3:
try:
trio.run(llfuse.main)
except:
return 1 # TODO return signal number if it was killed by signal
else:
return None
else:
return llfuse.main(workers=1)
# size of some LRUCaches (1 element per simultaneously open file)
@ -533,6 +558,7 @@ class FuseOperations(llfuse.Operations, FuseBackend):
finally:
llfuse.close(umount)
@async_wrapper
def statfs(self, ctx=None):
stat_ = llfuse.StatvfsData()
stat_.f_bsize = 512
@ -546,7 +572,7 @@ class FuseOperations(llfuse.Operations, FuseBackend):
stat_.f_namemax = 255 # == NAME_MAX (depends on archive source OS / FS)
return stat_
def getattr(self, inode, ctx=None):
def _getattr(self, inode, ctx=None):
item = self.get_item(inode)
entry = llfuse.EntryAttributes()
entry.st_ino = inode
@ -568,10 +594,16 @@ class FuseOperations(llfuse.Operations, FuseBackend):
entry.st_birthtime_ns = item.get('birthtime', mtime_ns)
return entry
@async_wrapper
def getattr(self, inode, ctx=None):
return self._getattr(inode, ctx=ctx)
@async_wrapper
def listxattr(self, inode, ctx=None):
item = self.get_item(inode)
return item.get('xattrs', {}).keys()
@async_wrapper
def getxattr(self, inode, name, ctx=None):
item = self.get_item(inode)
try:
@ -579,6 +611,7 @@ class FuseOperations(llfuse.Operations, FuseBackend):
except KeyError:
raise llfuse.FUSEError(llfuse.ENOATTR) from None
@async_wrapper
def lookup(self, parent_inode, name, ctx=None):
self.check_pending_archive(parent_inode)
if name == b'.':
@ -589,8 +622,9 @@ class FuseOperations(llfuse.Operations, FuseBackend):
inode = self.contents[parent_inode].get(name)
if not inode:
raise llfuse.FUSEError(errno.ENOENT)
return self.getattr(inode)
return self._getattr(inode)
@async_wrapper
def open(self, inode, flags, ctx=None):
if not self.allow_damaged_files:
item = self.get_item(inode)
@ -601,12 +635,14 @@ class FuseOperations(llfuse.Operations, FuseBackend):
logger.warning('File has damaged (all-zero) chunks. Try running borg check --repair. '
'Mount with allow_damaged_files to read damaged files.')
raise llfuse.FUSEError(errno.EIO)
return inode
return llfuse.FileInfo(fh=inode) if has_pyfuse3 else inode
@async_wrapper
def opendir(self, inode, ctx=None):
self.check_pending_archive(inode)
return inode
@async_wrapper
def read(self, fh, offset, size):
parts = []
item = self.get_item(fh)
@ -650,12 +686,25 @@ class FuseOperations(llfuse.Operations, FuseBackend):
break
return b''.join(parts)
def readdir(self, fh, off):
entries = [(b'.', fh), (b'..', self.parent[fh])]
entries.extend(self.contents[fh].items())
for i, (name, inode) in enumerate(entries[off:], off):
yield name, self.getattr(inode), i + 1
# note: we can't have a generator (with yield) and not a generator (async) in the same method
if has_pyfuse3:
async def readdir(self, fh, off, token):
entries = [(b'.', fh), (b'..', self.parent[fh])]
entries.extend(self.contents[fh].items())
for i, (name, inode) in enumerate(entries[off:], off):
attrs = self._getattr(inode)
if not llfuse.readdir_reply(token, name, attrs, i + 1):
break
else:
def readdir(self, fh, off):
entries = [(b'.', fh), (b'..', self.parent[fh])]
entries.extend(self.contents[fh].items())
for i, (name, inode) in enumerate(entries[off:], off):
attrs = self._getattr(inode)
yield name, attrs, i + 1
@async_wrapper
def readlink(self, inode, ctx=None):
item = self.get_item(inode)
return os.fsencode(item.source)

36
src/borg/fuse_impl.py Normal file
View File

@ -0,0 +1,36 @@
"""
load library for lowlevel FUSE implementation
"""
import os
BORG_FUSE_IMPL = os.environ.get('BORG_FUSE_IMPL', 'pyfuse3,llfuse')
for FUSE_IMPL in BORG_FUSE_IMPL.split(','):
FUSE_IMPL = FUSE_IMPL.strip()
if FUSE_IMPL == 'pyfuse3':
try:
import pyfuse3 as llfuse
except ImportError:
pass
else:
has_llfuse = False
has_pyfuse3 = True
break
elif FUSE_IMPL == 'llfuse':
try:
import llfuse
except ImportError:
pass
else:
has_llfuse = True
has_pyfuse3 = False
break
elif FUSE_IMPL == 'none':
pass
else:
raise RuntimeError("unknown fuse implementation in BORG_FUSE_IMPL: '%s'" % BORG_FUSE_IMPL)
else:
llfuse = None
has_llfuse = False
has_pyfuse3 = False

16
src/borg/testsuite/__init__.py
View File

@ -23,12 +23,10 @@ from .. import platform
# Note: this is used by borg.selftest, do not use or import py.test functionality here.
try:
import llfuse
# Does this version of llfuse support ns precision?
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
except ImportError:
have_fuse_mtime_ns = False
from ..fuse_impl import llfuse, has_pyfuse3, has_llfuse
# Does this version of llfuse support ns precision?
have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns') if llfuse else False
try:
from pytest import raises
@ -42,12 +40,6 @@ try:
except OSError:
has_lchflags = False
try:
import llfuse
has_llfuse = True or llfuse # avoids "unused import"
except ImportError:
has_llfuse = False
# The mtime get/set precision varies on different OS and Python versions
if posix and 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []):
st_mtime_ns_round = 0

23
src/borg/testsuite/archiver.py
View File

@ -26,11 +26,6 @@ from unittest.mock import patch
import pytest
try:
import llfuse
except ImportError:
pass
import borg
from .. import xattr, helpers, platform
from ..archive import Archive, ChunkBuffer
@ -55,7 +50,7 @@ from ..locking import LockFailed
from ..logger import setup_logging
from ..remote import RemoteRepository, PathNotAllowed
from ..repository import Repository
from . import has_lchflags, has_llfuse
from . import has_lchflags, llfuse
from . import BaseTestCase, changedir, environment_variable, no_selinux
from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported, is_birthtime_fully_supported
from .platform import fakeroot_detected
@ -798,7 +793,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
requires_hardlinks = pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported')
@requires_hardlinks
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_fuse_mount_hardlinks(self):
self._extract_hardlinks_setup()
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
@ -1661,7 +1656,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# verify that command works with read-only repo when using --bypass-lock
self.cmd('list', self.repository_location, '--bypass-lock')
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_readonly_mount(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.create_src_archive('test')
@ -1754,7 +1749,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# delete of the whole repository ignores features
self.cmd('delete', self.repository_location)
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_unknown_feature_on_mount(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
@ -2322,7 +2317,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert 'positional arguments' not in self.cmd('help', 'init', '--epilog-only')
assert 'This command initializes' not in self.cmd('help', 'init', '--usage-only')
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_fuse(self):
def has_noatime(some_file):
atime_before = os.stat(some_file).st_atime_ns
@ -2423,7 +2418,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
else:
raise
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_fuse_versions_view(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.create_regular_file('test', contents=b'first')
@ -2455,7 +2450,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert os.stat(hl2).st_ino == os.stat(hl3).st_ino
assert open(hl3, 'rb').read() == b'123456'
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_fuse_allow_damaged_files(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.create_src_archive('archive')
@ -2480,7 +2475,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
with self.fuse_mount(self.repository_location + '::archive', mountpoint, '-o', 'allow_damaged_files'):
open(os.path.join(mountpoint, path)).close()
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_fuse_mount_options(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
self.create_src_archive('arch11')
@ -2502,7 +2497,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
with self.fuse_mount(self.repository_location, mountpoint, '--prefix=nope'):
assert sorted(os.listdir(os.path.join(mountpoint))) == []
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
@unittest.skipUnless(llfuse, 'llfuse not installed')
def test_migrate_lock_alive(self):
"""Both old_id and new_id must not be stale during lock migration / daemonization."""
from functools import wraps

19
tox.ini
View File

@ -2,16 +2,29 @@
# fakeroot -u tox --recreate
[tox]
envlist = py{36,37,38,39},flake8
envlist = py{36,37,38,39}-fuse{2,3}, flake8
[testenv]
deps =
-rrequirements.d/development.txt
-rrequirements.d/fuse.txt
-rrequirements.d/development.txt
commands = py.test -v -n {env:XDISTN:1} -rs --cov=borg --cov-config=.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite}
# fakeroot -u needs some env vars:
passenv = *
[testenv:py{36,37,38,39}-fuse2]
setenv =
BORG_FUSE_IMPL=llfuse
deps =
llfuse
{[testenv]deps}
[testenv:py{36,37,38,39}-fuse3]
setenv =
BORG_FUSE_IMPL=pyfuse3
deps =
pyfuse3
{[testenv]deps}
[testenv:flake8]
changedir =
deps =