Merge pull request #8215 from ThomasWaldmann/fix-cythonize-import-error-reporting-master

setup.py: fix import error reporting for cythonize import, see #8208 (master)

.github/workflows/black.yaml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
             version: "~= 23.0"

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
     timeout-minutes: 5
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: chartboost/ruff-action@v1
 
   linux:
@@ -73,16 +73,16 @@ jobs:
     timeout-minutes: 120
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # just fetching 1 commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Cache pip
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
@@ -114,7 +114,7 @@ jobs:
         #sudo -E bash -c "tox -e py"
         tox --skip-missing-interpreters
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       env:
         OS: ${{ runner.os }}
         python: ${{ matrix.python-version }}
@@ -129,29 +129,29 @@ jobs:
       fail-fast: true
       matrix:
         include:
-            - os: macos-12
+            - os: macos-14
               python-version: '3.11'
               toxenv: py311-none  # note: no fuse testing, due to #6099, see also #6196.
 
     env:
       # Configure pkg-config to use OpenSSL from Homebrew
-      PKG_CONFIG_PATH: "/usr/local/opt/openssl@1.1/lib/pkgconfig:$PKG_CONFIG_PATH"
+      PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
       TOXENV: ${{ matrix.toxenv }}
 
     runs-on: ${{ matrix.os }}
     timeout-minutes: 180
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         # just fetching 1 commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Cache pip
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
@@ -170,21 +170,21 @@ jobs:
       env:
         # we already have that in the global env, but something is broken and overwrites that.
         # so, set it here, again.
-        PKG_CONFIG_PATH: "/usr/local/opt/openssl@1.1/lib/pkgconfig:$PKG_CONFIG_PATH"
+        PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
       run: |
-        pip install -e .
+        pip install -ve .
     - name: run tox env
       env:
         # we already have that in the global env, but something is broken and overwrites that.
         # so, set it here, again.
-        PKG_CONFIG_PATH: "/usr/local/opt/openssl@1.1/lib/pkgconfig:$PKG_CONFIG_PATH"
+        PKG_CONFIG_PATH: "/opt/homebrew/opt/openssl@3.0/lib/pkgconfig:$PKG_CONFIG_PATH"
         XDISTN: "6"
       run: |
         # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482.
         #sudo -E bash -c "tox -e py"
         tox --skip-missing-interpreters
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       env:
         OS: ${{ runner.os }}
         python: ${{ matrix.python-version }}
@@ -207,7 +207,7 @@ jobs:
         shell: msys2 {0}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: msys2/setup-msys2@v2
@@ -223,7 +223,7 @@ jobs:
           pyinstaller -y scripts/borg.exe.spec
           # build sdist and wheel in dist/...
           SETUPTOOLS_USE_DISTUTILS=stdlib python -m build
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: borg-windows
           path: dist/borg.exe

.github/workflows/codeql-analysis.yml

@@ -29,16 +29,16 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         # just fetching 1 commit is not enough for setuptools-scm, so we fetch all
         fetch-depth: 0
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
-        python-version: 3.9
+        python-version: 3.11
     - name: Cache pip
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
@@ -64,6 +64,6 @@ jobs:
         python3 -m venv ../borg-env
         source ../borg-env/bin/activate
         pip3 install -r requirements.d/development.txt
-        pip3 install -e .
+        pip3 install -ve .
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

Brewfile

@@ -2,7 +2,7 @@ brew 'pkg-config'
 brew 'zstd'
 brew 'lz4'
 brew 'xxhash'
-brew 'openssl@1.1'
+brew 'openssl@3.0'
 
 # osxfuse (aka macFUSE) is only required for "borg mount",
 # but won't work on github actions' workers.

Vagrantfile

@@ -28,8 +28,6 @@ def packages_debianoid(user)
     apt install -y python3-dev python3-setuptools virtualenv
     # for building python:
     apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
-    # older debian / ubuntu have no .pc file for these, so we need to point at the lib/header location:
-    echo 'export BORG_LIBXXHASH_PREFIX=/usr' >> ~vagrant/.bash_profile
   EOF
 end
 
@@ -132,11 +130,13 @@ def packages_openindiana
   return <<-EOF
     # needs separate provisioning step + reboot:
     #pkg update
-    #pkg install gcc-7 python-39 setuptools-39
+    pkg install gcc-13 git pkg-config libxxhash
     ln -sf /usr/bin/python3.9 /usr/bin/python3
     python3 -m ensurepip
     ln -sf /usr/bin/pip3.9 /usr/bin/pip3
     pip3 install virtualenv
+    # let borg's pkg-config find openssl:
+    pfexec pkg set-mediator -V 3.1 openssl
   EOF
 end
 
@@ -166,7 +166,7 @@ def install_pythons(boxname)
     . ~/.bash_profile
     echo "PYTHON_CONFIGURE_OPTS: ${PYTHON_CONFIGURE_OPTS}"
     pyenv install 3.12.0  # tests
-    pyenv install 3.11.7  # tests, binary build
+    pyenv install 3.11.8  # tests, binary build
     pyenv install 3.10.2  # tests
     pyenv install 3.9.4  # tests
     pyenv rehash
@@ -186,8 +186,8 @@ def build_pyenv_venv(boxname)
     . ~/.bash_profile
     cd /vagrant/borg
     # use the latest 3.11 release
-    pyenv global 3.11.7
-    pyenv virtualenv 3.11.7 borg-env
+    pyenv global 3.11.8
+    pyenv virtualenv 3.11.8 borg-env
     ln -s ~/.pyenv/versions/borg-env .
   EOF
 end
@@ -210,7 +210,7 @@ def install_pyinstaller()
     . ~/.bash_profile
     cd /vagrant/borg
     . borg-env/bin/activate
-    pip install 'pyinstaller==6.3.0'
+    pip install 'pyinstaller==6.5.0'
   EOF
 end
 
@@ -233,8 +233,8 @@ def run_tests(boxname, skip_env)
     . ../borg-env/bin/activate
     if which pyenv 2> /dev/null; then
       # for testing, use the earliest point releases of the supported python versions:
-      pyenv global 3.9.4 3.10.2 3.11.7 3.12.0
-      pyenv local 3.9.4 3.10.2 3.11.7 3.12.0
+      pyenv global 3.9.4 3.10.2 3.11.8 3.12.0
+      pyenv local 3.9.4 3.10.2 3.11.8 3.12.0
     fi
     # otherwise: just use the system python
     # some OSes can only run specific test envs, e.g. because they miss FUSE support:
@@ -275,16 +275,16 @@ Vagrant.configure(2) do |config|
     v.cpus = $cpus
   end
 
-  config.vm.define "lunar64" do |b|
-    b.vm.box = "ubuntu/lunar64"
+  config.vm.define "noble64" do |b|
+    b.vm.box = "ubuntu/noble64"
     b.vm.provider :virtualbox do |v|
       v.memory = 1024 + $wmem
     end
     b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
     b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("lunar64")
+    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("noble64")
     b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("lunar64", ".*none.*")
+    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("noble64", ".*none.*")
   end
 
   config.vm.define "jammy64" do |b|
@@ -331,22 +331,6 @@ Vagrant.configure(2) do |config|
     b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("bullseye64", ".*none.*")
   end
 
-  config.vm.define "buster64" do |b|
-    b.vm.box = "debian/buster64"
-    b.vm.provider :virtualbox do |v|
-      v.memory = 1024 + $wmem
-    end
-    b.vm.provision "fs init", :type => :shell, :inline => fs_init("vagrant")
-    b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid("vagrant")
-    b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("buster64")
-    b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("buster64")
-    b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("buster64")
-    b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("llfuse")
-    b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller()
-    b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("buster64")
-    b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("buster64", ".*none.*")
-  end
-
   config.vm.define "freebsd64" do |b|
     b.vm.box = "generic/freebsd14"
     b.vm.provider :virtualbox do |v|
@@ -417,7 +401,7 @@ Vagrant.configure(2) do |config|
   # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of
   # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again.
   config.vm.define "openindiana64" do |b|
-    b.vm.box = "openindiana"
+    b.vm.box = "openindiana/hipster"
     b.vm.provider :virtualbox do |v|
       v.memory = 2048 + $wmem
     end

docs/deployment.rst

@@ -14,3 +14,4 @@ This chapter details deployment strategies for the following scenarios.
    deployment/automated-local
    deployment/image-backup
    deployment/pull-backup
+   deployment/non-root-user

docs/deployment/non-root-user.rst

@@ -0,0 +1,66 @@
+.. include:: ../global.rst.inc
+.. highlight:: none
+.. _non_root_user:
+
+================================
+Backing up using a non-root user
+================================
+
+This section describes how to run borg as a non-root user and still be able to
+backup every file on the system.
+
+Normally borg is run as the root user to bypass all filesystem permissions and
+be able to read all files. But in theory this also allows borg to modify or
+delete files on your system, in case of a bug for example.
+
+To eliminate this possibility, we can run borg as a non-root user and give it read-only
+permissions to all files on the system.
+
+
+Using Linux capabilities inside a systemd service
+=================================================
+
+One way to do so, is to use linux `capabilities
+<https://man7.org/linux/man-pages/man7/capabilities.7.html>`_ within a systemd
+service.
+
+Linux capabilities allow us to give parts of the privileges the root user has to
+a non-root user. This works on a per-thread level and does not give the permission
+to the non-root user as a whole.
+
+For this we need to run our backup script from a systemd service and use the `AmbientCapabilities
+<https://www.freedesktop.org/software/systemd/man/latest/systemd.exec.html#AmbientCapabilities=>`_
+option added in systemd 229.
+
+A very basic unit file would look like this:
+
+::
+
+    [Unit]
+    Description=Borg Backup
+
+    [Service]
+    Type=oneshot
+    User=borg
+    ExecStart=/usr/local/sbin/backup.sh
+
+    AmbientCapabilities=CAP_DAC_READ_SEARCH
+
+The ``CAP_DAC_READ_SEARCH`` capability gives borg read-only access to all files and directories on the system.
+
+This service can then be started manually using ``systemctl start``, a systemd timer or other methods.
+
+Restore considerations
+======================
+
+When restoring files, the root user should be used. When using the non-root user, borg extract will
+change all files to be owned by the non-root user. Using borg mount will not allow the non-root user
+to access files that it would not have access to on the system itself.
+
+Other than that, the same restore process, that would be used when running the backup as root, can be used.
+
+.. warning::
+
+    When using a local repo and running borg commands as root, make sure to only use commands that do not
+    modify the repo itself, like extract or mount. Modifying the repo using the root user will break
+    the repo for the non-root user, since some files inside the repo will now be owned by root.

docs/global.rst.inc

@@ -16,7 +16,6 @@
 .. _libattr: https://savannah.nongnu.org/projects/attr/
 .. _liblz4: https://github.com/Cyan4973/lz4
 .. _libzstd: https://github.com/facebook/zstd
-.. _libb2: https://github.com/BLAKE2/libb2
 .. _OpenSSL: https://www.openssl.org/
 .. _`Python 3`: https://www.python.org/
 .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash

pyproject.toml

@@ -29,7 +29,7 @@ classifiers = [
 ]
 license = {text="BSD"}
 dependencies = [
-  "msgpack >=1.0.3, <=1.0.7",
+  "msgpack >=1.0.3, <=1.0.8",
   "packaging",
   "platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'",  # for macOS: breaking changes in 3.0.0,
   "platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'",  # for others: 2.6+ works consistently.
@@ -65,7 +65,7 @@ where = ["src"]
 "*" = ["*.c", "*.h", "*.pyx"]
 
 [build-system]
-requires = ["setuptools", "wheel", "pkgconfig", "Cython>=3", "setuptools_scm[toml]>=6.2"]
+requires = ["setuptools", "wheel", "pkgconfig", "Cython>=3.0.3", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]

requirements.d/development.lock.txt

@@ -1,14 +1,14 @@
-setuptools==68.2.2
+setuptools==69.2.0
 setuptools-scm==8.0.4
-pip==23.3.2
-wheel==0.41.3
-virtualenv==20.25.0
-build==1.0.3
+pip==24.0
+wheel==0.43.0
+virtualenv==20.25.1
+build==1.2.1
 pkgconfig==1.5.5
-tox==4.11.3
-pytest==7.4.3
-pytest-xdist==3.3.1
-pytest-cov==4.1.0
+tox==4.14.2
+pytest==8.1.1
+pytest-xdist==3.5.0
+pytest-cov==5.0.0
 pytest-benchmark==4.0.0
-Cython==3.0.5
-pre-commit==3.5.0
+Cython==3.0.10
+pre-commit==3.7.0

setup.py

@@ -16,8 +16,14 @@ from setuptools.command.sdist import sdist
 
 try:
     from Cython.Build import cythonize
-except ImportError:
+
+    cythonize_import_error_msg = None
+except ImportError as exc:
+    # either there is no Cython installed or there is some issue with it.
     cythonize = None
+    cythonize_import_error_msg = "ImportError: " + str(exc)
+    if "failed to map segment from shared object" in cythonize_import_error_msg:
+        cythonize_import_error_msg += " Check if the borg build uses a +exec filesystem."
 
 sys.path += [os.path.dirname(__file__)]
 
@@ -80,7 +86,12 @@ else:
 
     cython_c_files = [fn.replace(".pyx", ".c") for fn in cython_sources]
     if not on_rtd and not all(os.path.exists(path) for path in cython_c_files):
-        raise ImportError("The GIT version of Borg needs Cython. Install Cython or use a released version.")
+        raise ImportError(
+            "The GIT version of Borg needs a working Cython. "
+            + "Install or fix Cython or use a released borg version. "
+            + "Importing cythonize failed with: "
+            + cythonize_import_error_msg
+        )
 
 
 cmdclass = {"build_ext": build_ext, "sdist": Sdist}

src/borg/archive.py

@@ -199,6 +199,7 @@ class BackupIO:
             E_MAP = {
                 errno.EPERM: BackupPermissionError,
                 errno.EACCES: BackupPermissionError,
+                errno.EBUSY: BackupPermissionError,
                 errno.ENOENT: BackupFileNotFoundError,
                 errno.EIO: BackupIOError,
             }
@@ -963,7 +964,11 @@ Duration: {0.duration}
                     if not symlink:
                         os.chmod(path, item.mode)
             if not self.noacls:
-                acl_set(path, item, self.numeric_ids, fd=fd)
+                try:
+                    acl_set(path, item, self.numeric_ids, fd=fd)
+                except OSError as e:
+                    if e.errno not in (errno.ENOTSUP,):
+                        raise
             if not self.noxattrs and "xattrs" in item:
                 # chown removes Linux capabilities, so set the extended attributes at the end, after chown,
                 # since they include the Linux capabilities in the "security.capability" attribute.
@@ -1209,7 +1214,11 @@ class MetadataCollector:
             attrs["xattrs"] = StableDict(xattrs)
         if not self.noacls:
             with backup_io("extended stat (ACLs)"):
-                acl_get(path, attrs, st, self.numeric_ids, fd=fd)
+                try:
+                    acl_get(path, attrs, st, self.numeric_ids, fd=fd)
+                except OSError as e:
+                    if e.errno not in (errno.ENOTSUP,):
+                        raise
         return attrs
 
     def stat_attrs(self, st, path, fd=None):

src/borg/archiver/benchmark_cmd.py

@@ -18,13 +18,22 @@ class BenchmarkMixIn:
     def do_benchmark_crud(self, args):
         """Benchmark Create, Read, Update, Delete for archives."""
 
+        def parse_args(args, cmd):
+            # we need to inherit some essential options from the "borg benchmark crud" invocation
+            if args.rsh is not None:
+                cmd[1:1] = ["--rsh", args.rsh]
+            if args.remote_path is not None:
+                cmd[1:1] = ["--remote-path", args.remote_path]
+            return self.parse_args(cmd)
+
         def measurement_run(repo, path):
             compression = "--compression=none"
             # measure create perf (without files cache to always have it chunking)
             t_start = time.monotonic()
             rc = get_reset_ec(
                 self.do_create(
-                    self.parse_args(
+                    parse_args(
+                        args,
                         [
                             f"--repo={repo}",
                             "create",
@@ -32,7 +41,7 @@ class BenchmarkMixIn:
                             "--files-cache=disabled",
                             "borg-benchmark-crud1",
                             path,
-                        ]
+                        ],
                     )
                 )
             )
@@ -41,27 +50,31 @@ class BenchmarkMixIn:
             assert rc == 0
             # now build files cache
             rc1 = get_reset_ec(
-                self.do_create(self.parse_args([f"--repo={repo}", "create", compression, "borg-benchmark-crud2", path]))
+                self.do_create(
+                    parse_args(args, [f"--repo={repo}", "create", compression, "borg-benchmark-crud2", path])
+                )
             )
             rc2 = get_reset_ec(
-                self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud2"]))
+                self.do_delete(parse_args(args, [f"--repo={repo}", "delete", "-a", "borg-benchmark-crud2"]))
             )
             assert rc1 == rc2 == 0
             # measure a no-change update (archive1 is still present)
             t_start = time.monotonic()
             rc1 = get_reset_ec(
-                self.do_create(self.parse_args([f"--repo={repo}", "create", compression, "borg-benchmark-crud3", path]))
+                self.do_create(
+                    parse_args(args, [f"--repo={repo}", "create", compression, "borg-benchmark-crud3", path])
+                )
             )
             t_end = time.monotonic()
             dt_update = t_end - t_start
             rc2 = get_reset_ec(
-                self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud3"]))
+                self.do_delete(parse_args(args, [f"--repo={repo}", "delete", "-a", "borg-benchmark-crud3"]))
             )
             assert rc1 == rc2 == 0
             # measure extraction (dry-run: without writing result to disk)
             t_start = time.monotonic()
             rc = get_reset_ec(
-                self.do_extract(self.parse_args([f"--repo={repo}", "extract", "borg-benchmark-crud1", "--dry-run"]))
+                self.do_extract(parse_args(args, [f"--repo={repo}", "extract", "borg-benchmark-crud1", "--dry-run"]))
             )
             t_end = time.monotonic()
             dt_extract = t_end - t_start
@@ -69,7 +82,7 @@ class BenchmarkMixIn:
             # measure archive deletion (of LAST present archive with the data)
             t_start = time.monotonic()
             rc = get_reset_ec(
-                self.do_delete(self.parse_args([f"--repo={repo}", "delete", "-a", "borg-benchmark-crud1"]))
+                self.do_delete(parse_args(args, [f"--repo={repo}", "delete", "-a", "borg-benchmark-crud1"]))
             )
             t_end = time.monotonic()
             dt_delete = t_end - t_start
@@ -226,8 +239,6 @@ class BenchmarkMixIn:
         spec = "msgpack"
         print(f"{spec:<12} {size:<10} {timeit(lambda: msgpack.packb(items), number=100):.3f}s")
 
-        return 0
-
     def build_parser_benchmarks(self, subparsers, common_parser, mid_common_parser):
         from ._common import process_epilog
 

src/borg/archiver/prune_cmd.py

@@ -169,7 +169,7 @@ class PruneMixIn:
                     or (args.list_pruned and archive in to_delete)
                     or (args.list_kept and archive not in to_delete)
                 ):
-                    list_logger.info(f"{log_message:<40} {formatter.format_item(archive, jsonline=False)}")
+                    list_logger.info(f"{log_message:<44} {formatter.format_item(archive, jsonline=False)}")
             pi.finish()
             if sig_int:
                 # Ctrl-C / SIGINT: do not checkpoint (commit) again, we already have a checkpoint in this case.

src/borg/helpers/msgpack.py

@@ -209,7 +209,7 @@ def is_supported_msgpack():
 
     if msgpack.version in []:  # < add bad releases here to deny list
         return False
-    return (1, 0, 3) <= msgpack.version <= (1, 0, 7)
+    return (1, 0, 3) <= msgpack.version <= (1, 0, 8)
 
 
 def get_limited_unpacker(kind):

src/borg/helpers/process.py

@@ -195,6 +195,8 @@ class SigIntManager:
         self._action_triggered = False
         self._action_done = False
         self.ctx = signal_handler("SIGINT", self.handler)
+        self.debounce_interval = 20000000  # ns
+        self.last = None  # monotonic time when we last processed SIGINT
 
     def __bool__(self):
         # this will be True (and stay True) after the first Ctrl-C/SIGINT
@@ -215,10 +217,22 @@ class SigIntManager:
         self._action_done = True
 
     def handler(self, sig_no, stack):
-        # handle the first ctrl-c / SIGINT.
-        self.__exit__(None, None, None)
-        self._sig_int_triggered = True
-        self._action_triggered = True
+        # Ignore a SIGINT if it comes too quickly after the last one, e.g. because it
+        # was caused by the same Ctrl-C key press and a parent process forwarded it to us.
+        # This can easily happen for the pyinstaller-made binaries because the bootloader
+        # process and the borg process are in same process group (see #8155), but maybe also
+        # under other circumstances.
+        now = time.monotonic_ns()
+        if self.last is None:  # first SIGINT
+            self.last = now
+            self._sig_int_triggered = True
+            self._action_triggered = True
+        elif now - self.last >= self.debounce_interval:  # second SIGINT
+            # restore the original signal handler for the 3rd+ SIGINT -
+            # this implies that this handler here loses control!
+            self.__exit__(None, None, None)
+            # handle 2nd SIGINT like the default handler would do it:
+            raise KeyboardInterrupt  # python docs say this might show up at an arbitrary place.
 
     def __enter__(self):
         self.ctx.__enter__()

src/borg/helpers/time.py

@@ -100,7 +100,7 @@ def format_timedelta(td):
     s = ts % 60
     m = int(ts / 60) % 60
     h = int(ts / 3600) % 24
-    txt = "%.2f seconds" % s
+    txt = "%.3f seconds" % s
     if m:
         txt = "%d minutes %s" % (m, txt)
     if h:

src/borg/platform/darwin.pyx

@@ -1,6 +1,7 @@
 import os
 
 from libc.stdint cimport uint32_t
+from libc cimport errno
 
 from .posix import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
@@ -115,20 +116,25 @@ def _remove_non_numeric_identifier(acl):
 def acl_get(path, item, st, numeric_ids=False, fd=None):
     cdef acl_t acl = NULL
     cdef char *text = NULL
+    if isinstance(path, str):
+        path = os.fsencode(path)
     try:
         if fd is not None:
             acl = acl_get_fd_np(fd, ACL_TYPE_EXTENDED)
         else:
-            if isinstance(path, str):
-                path = os.fsencode(path)
             acl = acl_get_link_np(path, ACL_TYPE_EXTENDED)
-        if acl is not NULL:
-            text = acl_to_text(acl, NULL)
-            if text is not NULL:
-                if numeric_ids:
-                    item['acl_extended'] = _remove_non_numeric_identifier(text)
-                else:
-                    item['acl_extended'] = text
+        if acl == NULL:
+            if errno.errno == errno.ENOENT:
+                # macOS weirdness: if a file has no ACLs, it sets errno to ENOENT. :-(
+                return
+            raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+        text = acl_to_text(acl, NULL)
+        if text == NULL:
+            raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+        if numeric_ids:
+            item['acl_extended'] = _remove_non_numeric_identifier(text)
+        else:
+            item['acl_extended'] = text
     finally:
         acl_free(text)
         acl_free(acl)
@@ -139,16 +145,19 @@ def acl_set(path, item, numeric_ids=False, fd=None):
     acl_text = item.get('acl_extended')
     if acl_text is not None:
         try:
+            if isinstance(path, str):
+                path = os.fsencode(path)
             if numeric_ids:
                 acl = acl_from_text(acl_text)
             else:
                 acl = acl_from_text(<bytes>_remove_numeric_id_if_possible(acl_text))
-            if acl is not NULL:
-                if fd is not None:
-                    acl_set_fd_np(fd, acl, ACL_TYPE_EXTENDED)
-                else:
-                    if isinstance(path, str):
-                        path = os.fsencode(path)
-                    acl_set_link_np(path, ACL_TYPE_EXTENDED, acl)
+            if acl == NULL:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            if fd is not None:
+                if acl_set_fd_np(fd, acl, ACL_TYPE_EXTENDED) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            else:
+                if acl_set_link_np(path, ACL_TYPE_EXTENDED, acl) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
         finally:
             acl_free(acl)

src/borg/platform/freebsd.pyx

@@ -1,4 +1,7 @@
 import os
+import stat
+
+from libc cimport errno
 
 from .posix import posix_acl_use_stored_uid_gid
 from ..helpers import safe_encode, safe_decode
@@ -6,10 +9,6 @@ from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_lst
 
 API_VERSION = '1.2_05'
 
-cdef extern from "errno.h":
-    int errno
-    int EINVAL
-
 cdef extern from "sys/extattr.h":
     ssize_t c_extattr_list_file "extattr_list_file" (const char *path, int attrnamespace, void *data, size_t nbytes)
     ssize_t c_extattr_list_link "extattr_list_link" (const char *path, int attrnamespace, void *data, size_t nbytes)
@@ -44,10 +43,12 @@ cdef extern from "sys/acl.h":
     char *acl_to_text_np(acl_t acl, ssize_t *len, int flags)
     int ACL_TEXT_NUMERIC_IDS
     int ACL_TEXT_APPEND_ID
+    int acl_extended_link_np(const char * path)  # check also: acl_is_trivial_np
 
 cdef extern from "unistd.h":
     long lpathconf(const char *path, int name)
     int _PC_ACL_NFS4
+    int _PC_ACL_EXTENDED
 
 
 # On FreeBSD, borg currently only deals with the USER namespace as it is unclear
@@ -124,21 +125,21 @@ def setxattr(path, name, value, *, follow_symlinks=False):
 
 
 cdef _get_acl(p, type, item, attribute, flags, fd=None):
-    cdef acl_t acl = NULL
-    cdef char *text = NULL
-    try:
-        if fd is not None:
-            acl = acl_get_fd_np(fd, type)
-        else:
-            acl = acl_get_link_np(p, type)
-        if acl is not NULL:
-            text = acl_to_text_np(acl, NULL, flags)
-            if text is not NULL:
-                item[attribute] = text
-    finally:
-        acl_free(text)
+    cdef acl_t acl
+    cdef char *text
+    if fd is not None:
+        acl = acl_get_fd_np(fd, type)
+    else:
+        acl = acl_get_link_np(p, type)
+    if acl == NULL:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(p))
+    text = acl_to_text_np(acl, NULL, flags)
+    if text == NULL:
         acl_free(acl)
-
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(p))
+    item[attribute] = text
+    acl_free(text)
+    acl_free(acl)
 
 def acl_get(path, item, st, numeric_ids=False, fd=None):
     """Saves ACL Entries
@@ -146,34 +147,46 @@ def acl_get(path, item, st, numeric_ids=False, fd=None):
     If `numeric_ids` is True the user/group field is not preserved only uid/gid
     """
     cdef int flags = ACL_TEXT_APPEND_ID
+    flags |= ACL_TEXT_NUMERIC_IDS if numeric_ids else 0
     if isinstance(path, str):
         path = os.fsencode(path)
-    ret = lpathconf(path, _PC_ACL_NFS4)
-    if ret < 0 and errno == EINVAL:
+    ret = acl_extended_link_np(path)
+    if ret < 0:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+    if ret == 0:
+        # there is no ACL defining permissions other than those defined by the traditional file permission bits.
         return
-    flags |= ACL_TEXT_NUMERIC_IDS if numeric_ids else 0
-    if ret > 0:
+    ret = lpathconf(path, _PC_ACL_NFS4)
+    if ret < 0:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+    nfs4_acl = ret == 1
+    if nfs4_acl:
         _get_acl(path, ACL_TYPE_NFS4, item, 'acl_nfs4', flags, fd=fd)
     else:
         _get_acl(path, ACL_TYPE_ACCESS, item, 'acl_access', flags, fd=fd)
-        _get_acl(path, ACL_TYPE_DEFAULT, item, 'acl_default', flags, fd=fd)
+        if stat.S_ISDIR(st.st_mode):
+            _get_acl(path, ACL_TYPE_DEFAULT, item, 'acl_default', flags, fd=fd)
 
 
 cdef _set_acl(path, type, item, attribute, numeric_ids=False, fd=None):
     cdef acl_t acl = NULL
     text = item.get(attribute)
-    if text is not None:
-        if numeric_ids and type == ACL_TYPE_NFS4:
-            text = _nfs4_use_stored_uid_gid(text)
-        elif numeric_ids and type in (ACL_TYPE_ACCESS, ACL_TYPE_DEFAULT):
-            text = posix_acl_use_stored_uid_gid(text)
+    if text:
+        if numeric_ids:
+            if type == ACL_TYPE_NFS4:
+                text = _nfs4_use_stored_uid_gid(text)
+            elif type in (ACL_TYPE_ACCESS, ACL_TYPE_DEFAULT):
+                text = posix_acl_use_stored_uid_gid(text)
+        acl = acl_from_text(<bytes>text)
+        if acl == NULL:
+            raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
         try:
-            acl = acl_from_text(<bytes> text)
-            if acl is not NULL:
-                if fd is not None:
-                    acl_set_fd_np(fd, acl, type)
-                else:
-                    acl_set_link_np(path, type, acl)
+            if fd is not None:
+                if acl_set_fd_np(fd, acl, type) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            else:
+                if acl_set_link_np(path, type, acl) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
         finally:
             acl_free(acl)
 
@@ -201,6 +214,14 @@ def acl_set(path, item, numeric_ids=False, fd=None):
     """
     if isinstance(path, str):
         path = os.fsencode(path)
-    _set_acl(path, ACL_TYPE_NFS4, item, 'acl_nfs4', numeric_ids, fd=fd)
-    _set_acl(path, ACL_TYPE_ACCESS, item, 'acl_access', numeric_ids, fd=fd)
-    _set_acl(path, ACL_TYPE_DEFAULT, item, 'acl_default', numeric_ids, fd=fd)
+    ret = lpathconf(path, _PC_ACL_NFS4)
+    if ret < 0:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+    if ret == 1:
+        _set_acl(path, ACL_TYPE_NFS4, item, 'acl_nfs4', numeric_ids, fd=fd)
+    ret = lpathconf(path, _PC_ACL_EXTENDED)
+    if ret < 0:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+    if ret == 1:
+        _set_acl(path, ACL_TYPE_ACCESS, item, 'acl_access', numeric_ids, fd=fd)
+        _set_acl(path, ACL_TYPE_DEFAULT, item, 'acl_default', numeric_ids, fd=fd)

src/borg/platform/linux.pyx

@@ -50,7 +50,7 @@ cdef extern from "sys/acl.h":
     char *acl_to_text(acl_t acl, ssize_t *len)
 
 cdef extern from "acl/libacl.h":
-    int acl_extended_file(const char *path)
+    int acl_extended_file_nofollow(const char *path)
     int acl_extended_fd(int fd)
 
 cdef extern from "linux/fs.h":
@@ -233,15 +233,19 @@ def acl_get(path, item, st, numeric_ids=False, fd=None):
     cdef acl_t access_acl = NULL
     cdef char *default_text = NULL
     cdef char *access_text = NULL
+    cdef int ret = 0
 
-    if stat.S_ISLNK(st.st_mode):
-        # symlinks can not have ACLs
-        return
     if isinstance(path, str):
         path = os.fsencode(path)
-    if (fd is not None and acl_extended_fd(fd) <= 0
-        or
-        fd is None and acl_extended_file(path) <= 0):
+    if fd is not None:
+        ret = acl_extended_fd(fd)
+    else:
+        ret = acl_extended_file_nofollow(path)
+    if ret < 0:
+        raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+    if ret == 0:
+        # there is no ACL defining permissions other than those defined by the traditional file permission bits.
+        # note: this should also be the case for symlink fs objects, as they can not have ACLs.
         return
     if numeric_ids:
         converter = acl_numeric_ids
@@ -252,25 +256,28 @@ def acl_get(path, item, st, numeric_ids=False, fd=None):
             access_acl = acl_get_fd(fd)
         else:
             access_acl = acl_get_file(path, ACL_TYPE_ACCESS)
-        if access_acl is not NULL:
-            access_text = acl_to_text(access_acl, NULL)
-            if access_text is not NULL:
-                item['acl_access'] = converter(access_text)
+        if access_acl == NULL:
+            raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+        access_text = acl_to_text(access_acl, NULL)
+        if access_text == NULL:
+            raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+        item['acl_access'] = converter(access_text)
     finally:
         acl_free(access_text)
         acl_free(access_acl)
-
-    try:
-        if stat.S_ISDIR(st.st_mode):
-            # only directories can have a default ACL. there is no fd-based api to get it.
+    if stat.S_ISDIR(st.st_mode):
+        # only directories can have a default ACL. there is no fd-based api to get it.
+        try:
             default_acl = acl_get_file(path, ACL_TYPE_DEFAULT)
-            if default_acl is not NULL:
-                default_text = acl_to_text(default_acl, NULL)
-                if default_text is not NULL:
-                    item['acl_default'] = converter(default_text)
-    finally:
-        acl_free(default_text)
-        acl_free(default_acl)
+            if default_acl == NULL:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            default_text = acl_to_text(default_acl, NULL)
+            if default_text == NULL:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            item['acl_default'] = converter(default_text)
+        finally:
+            acl_free(default_text)
+            acl_free(default_acl)
 
 
 def acl_set(path, item, numeric_ids=False, fd=None):
@@ -281,7 +288,7 @@ def acl_set(path, item, numeric_ids=False, fd=None):
         # Linux does not support setting ACLs on symlinks
         return
 
-    if fd is None and isinstance(path, str):
+    if isinstance(path, str):
         path = os.fsencode(path)
     if numeric_ids:
         converter = posix_acl_use_stored_uid_gid
@@ -290,21 +297,26 @@ def acl_set(path, item, numeric_ids=False, fd=None):
     access_text = item.get('acl_access')
     if access_text is not None:
         try:
-            access_acl = acl_from_text(<bytes> converter(access_text))
-            if access_acl is not NULL:
-                if fd is not None:
-                    acl_set_fd(fd, access_acl)
-                else:
-                    acl_set_file(path, ACL_TYPE_ACCESS, access_acl)
+            access_acl = acl_from_text(<bytes>converter(access_text))
+            if access_acl == NULL:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            if fd is not None:
+                if acl_set_fd(fd, access_acl) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            else:
+                if acl_set_file(path, ACL_TYPE_ACCESS, access_acl) == -1:
+                    raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
         finally:
             acl_free(access_acl)
     default_text = item.get('acl_default')
     if default_text is not None:
         try:
-            default_acl = acl_from_text(<bytes> converter(default_text))
-            if default_acl is not NULL:
-                # only directories can get a default ACL. there is no fd-based api to set it.
-                acl_set_file(path, ACL_TYPE_DEFAULT, default_acl)
+            default_acl = acl_from_text(<bytes>converter(default_text))
+            if default_acl == NULL:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
+            # only directories can get a default ACL. there is no fd-based api to set it.
+            if acl_set_file(path, ACL_TYPE_DEFAULT, default_acl) == -1:
+                raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path))
         finally:
             acl_free(default_acl)
 

src/borg/testsuite/archive.py

@@ -61,8 +61,8 @@ def test_stats_format(stats):
 Number of files: 1
 Original size: 20 B
 Deduplicated size: 20 B
-Time spent in hashing: 0.00 seconds
-Time spent in chunking: 0.00 seconds
+Time spent in hashing: 0.000 seconds
+Time spent in chunking: 0.000 seconds
 Added files: 0
 Unchanged files: 0
 Modified files: 0

src/borg/testsuite/helpers.py

@@ -368,7 +368,7 @@ def test_text_invalid(text):
 def test_format_timedelta():
     t0 = datetime(2001, 1, 1, 10, 20, 3, 0)
     t1 = datetime(2001, 1, 1, 12, 20, 4, 100000)
-    assert format_timedelta(t1 - t0) == "2 hours 1.10 seconds"
+    assert format_timedelta(t1 - t0) == "2 hours 1.100 seconds"
 
 
 @pytest.mark.parametrize(

src/borg/testsuite/platform.py

@@ -1,3 +1,4 @@
+import errno
 import functools
 import os
 
@@ -31,25 +32,38 @@ def are_acls_working():
     with unopened_tempfile() as filepath:
         open(filepath, "w").close()
         try:
-            if is_freebsd:
-                access = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-\n"
-                contained = b"user:root:rw-"
+            if is_darwin:
+                acl_key = "acl_extended"
+                acl_value = b"!#acl 1\nuser:FFFFEEEE-DDDD-CCCC-BBBB-AAAA00000000:root:0:allow:read\n"
             elif is_linux:
-                access = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:0\n"
-                contained = b"user:root:rw-:0"
-            elif is_darwin:
-                return True  # improve?
+                acl_key = "acl_access"
+                acl_value = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n"
+            elif is_freebsd:
+                acl_key = "acl_access"
+                acl_value = b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-\ngroup:wheel:rw-\n"
             else:
-                return False  # unsupported platform
-            acl = {"acl_access": access}
-            acl_set(filepath, acl)
+                return False  # ACLs unsupported on this platform.
+            write_acl = {acl_key: acl_value}
+            acl_set(filepath, write_acl)
             read_acl = {}
             acl_get(filepath, read_acl, os.stat(filepath))
-            read_acl_access = read_acl.get("acl_access", None)
-            if read_acl_access and contained in read_acl_access:
-                return True
+            acl = read_acl.get(acl_key, None)
+            if acl is not None:
+                if is_darwin:
+                    check_for = b"root:0:allow:read"
+                elif is_linux:
+                    check_for = b"user::rw-"
+                elif is_freebsd:
+                    check_for = b"user::rw-"
+                else:
+                    return False  # ACLs unsupported on this platform.
+                if check_for in acl:
+                    return True
         except PermissionError:
             pass
+        except OSError as e:
+            if e.errno not in (errno.ENOTSUP,):
+                raise
         return False
 
 

src/borg/testsuite/platform_darwin.py

@@ -20,7 +20,7 @@ def set_acl(path, acl, numeric_ids=False):
 
 
 @skipif_acls_not_working
-def test_access_acl():
+def test_extended_acl():
     file = tempfile.NamedTemporaryFile()
     assert get_acl(file.name) == {}
     set_acl(

src/borg/testsuite/platform_freebsd.py

@@ -49,6 +49,7 @@ def set_acl(path, access=None, default=None, nfs4=None, numeric_ids=False):
 @skipif_acls_not_working
 def test_access_acl():
     file1 = tempfile.NamedTemporaryFile()
+    assert get_acl(file1.name) == {}
     set_acl(
         file1.name,
         access=b"user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-\ngroup:wheel:rw-\n",
@@ -86,6 +87,7 @@ def test_access_acl():
 @skipif_acls_not_working
 def test_default_acl():
     tmpdir = tempfile.mkdtemp()
+    assert get_acl(tmpdir) == {}
     set_acl(tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL)
     assert get_acl(tmpdir)["acl_access"] == ACCESS_ACL
     assert get_acl(tmpdir)["acl_default"] == DEFAULT_ACL