Merge pull request #7300 from RayyanAnsari/borg-platformdirs

use platformdirs

conftest.py

@@ -20,13 +20,12 @@ from borg.testsuite.platform import fakeroot_detected  # noqa: E402
 
 @pytest.fixture(autouse=True)
 def clean_env(tmpdir_factory, monkeypatch):
-    # avoid that we access / modify the user's normal .config / .cache directory:
-    monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir_factory.mktemp("xdg-config-home")))
-    monkeypatch.setenv("XDG_CACHE_HOME", str(tmpdir_factory.mktemp("xdg-cache-home")))
     # also avoid to use anything from the outside environment:
     keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)]
     for key in keys:
         monkeypatch.delenv(key, raising=False)
+    # avoid that we access / modify the user's normal .config / .cache directory:
+    monkeypatch.setenv("BORG_BASE_DIR", str(tmpdir_factory.mktemp("borg-base-dir")))
     # Speed up tests
     monkeypatch.setenv("BORG_TESTONLY_WEAKEN_KDF", "1")
 

scripts/msys2-install-deps

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,python,cython,python-setuptools,python-wheel,python-pkgconfig,python-packaging,python-msgpack,python-argon2_cffi,python-pip}
+pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-pkgconfig,python-packaging,python-pip}
 pip install pyinstaller
 
 if [ "$1" = "development" ]; then

setup.cfg

@@ -41,6 +41,7 @@ setup_requires =
 install_requires =
   msgpack >=1.0.3, <=1.0.4
   packaging
+  platformdirs >=3.0.0, <4.0.0
   argon2-cffi
 tests_require =
   pytest

src/borg/helpers/__init__.py

@@ -11,7 +11,7 @@ from ..constants import *  # NOQA
 from .checks import check_extension_modules, check_python
 from .datastruct import StableDict, Buffer, EfficientCollectionQueue
 from .errors import Error, ErrorWithTraceback, IntegrityError, DecompressionError
-from .fs import ensure_dir, get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir
+from .fs import ensure_dir, get_security_dir, get_keys_dir, get_base_dir, join_base_dir, get_cache_dir, get_config_dir
 from .fs import dir_is_tagged, dir_is_cachedir, make_path_safe, scandir_inorder
 from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, umount
 from .fs import O_, flags_root, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime

src/borg/helpers/fs.py

@@ -8,6 +8,8 @@ import subprocess
 import sys
 import textwrap
 
+import platformdirs
+
 from .errors import Error
 
 from .process import prepare_subprocess_env
@@ -40,7 +42,7 @@ def ensure_dir(path, mode=stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO, pretty_dea
             raise
 
 
-def get_base_dir():
+def get_base_dir(*, legacy=False):
     """Get home directory / base directory for borg:
 
     - BORG_BASE_DIR, if set
@@ -48,47 +50,65 @@ def get_base_dir():
     - ~$USER, if USER is set
     - ~
     """
-    base_dir = os.environ.get("BORG_BASE_DIR") or os.environ.get("HOME")
+    if legacy:
-    # os.path.expanduser() behaves differently for '~' and '~someuser' as
+        base_dir = os.environ.get("BORG_BASE_DIR") or os.environ.get("HOME")
-    # parameters: when called with an explicit username, the possibly set
+        # os.path.expanduser() behaves differently for '~' and '~someuser' as
-    # environment variable HOME is no longer respected. So we have to check if
+        # parameters: when called with an explicit username, the possibly set
-    # it is set and only expand the user's home directory if HOME is unset.
+        # environment variable HOME is no longer respected. So we have to check if
-    if not base_dir:
+        # it is set and only expand the user's home directory if HOME is unset.
-        base_dir = os.path.expanduser("~%s" % os.environ.get("USER", ""))
+        if not base_dir:
+            base_dir = os.path.expanduser("~%s" % os.environ.get("USER", ""))
+    else:
+        # we only care for BORG_BASE_DIR here, as it can be used to override the base dir
+        # and not use any more or less platform specific way to determine the base dir.
+        base_dir = os.environ.get("BORG_BASE_DIR")
     return base_dir
 
 
-def get_keys_dir():
+def join_base_dir(*paths, **kw):
+    legacy = kw.get("legacy", True)
+    base_dir = get_base_dir(legacy=legacy)
+    return None if base_dir is None else os.path.join(base_dir, *paths)
+
+
+def get_keys_dir(*, legacy=False):
     """Determine where to repository keys and cache"""
     keys_dir = os.environ.get("BORG_KEYS_DIR")
     if keys_dir is None:
         # note: do not just give this as default to the environment.get(), see issue #5979.
-        keys_dir = os.path.join(get_config_dir(), "keys")
+        keys_dir = os.path.join(get_config_dir(legacy=legacy), "keys")
     ensure_dir(keys_dir)
     return keys_dir
 
 
-def get_security_dir(repository_id=None):
+def get_security_dir(repository_id=None, *, legacy=False):
     """Determine where to store local security information."""
     security_dir = os.environ.get("BORG_SECURITY_DIR")
     if security_dir is None:
         # note: do not just give this as default to the environment.get(), see issue #5979.
-        security_dir = os.path.join(get_config_dir(), "security")
+        security_dir = os.path.join(get_config_dir(legacy=legacy), "security")
     if repository_id:
         security_dir = os.path.join(security_dir, repository_id)
     ensure_dir(security_dir)
     return security_dir
 
 
-def get_cache_dir():
+def get_cache_dir(*, legacy=False):
     """Determine where to repository keys and cache"""
-    # Get cache home path
+
-    cache_home = os.path.join(get_base_dir(), ".cache")
+    if legacy:
-    # Try to use XDG_CACHE_HOME instead if BORG_BASE_DIR isn't explicitly set
+        # Get cache home path
-    if not os.environ.get("BORG_BASE_DIR"):
+        cache_home = join_base_dir(".cache", legacy=legacy)
-        cache_home = os.environ.get("XDG_CACHE_HOME", cache_home)
+        # Try to use XDG_CACHE_HOME instead if BORG_BASE_DIR isn't explicitly set
-    # Use BORG_CACHE_DIR if set, otherwise assemble final path from cache home path
+        if not os.environ.get("BORG_BASE_DIR"):
-    cache_dir = os.environ.get("BORG_CACHE_DIR", os.path.join(cache_home, "borg"))
+            cache_home = os.environ.get("XDG_CACHE_HOME", cache_home)
+        # Use BORG_CACHE_DIR if set, otherwise assemble final path from cache home path
+        cache_dir = os.environ.get("BORG_CACHE_DIR", os.path.join(cache_home, "borg"))
+    else:
+        cache_dir = os.environ.get(
+            "BORG_CACHE_DIR", join_base_dir(".cache", legacy=legacy) or platformdirs.user_cache_dir("borg")
+        )
+
     # Create path if it doesn't exist yet
     ensure_dir(cache_dir)
     cache_tag_fn = os.path.join(cache_dir, CACHE_TAG_NAME)
@@ -110,15 +130,22 @@ def get_cache_dir():
     return cache_dir
 
 
-def get_config_dir():
+def get_config_dir(*, legacy=False):
     """Determine where to store whole config"""
+
     # Get config home path
-    config_home = os.path.join(get_base_dir(), ".config")
+    if legacy:
-    # Try to use XDG_CONFIG_HOME instead if BORG_BASE_DIR isn't explicitly set
+        config_home = join_base_dir(".config", legacy=legacy)
-    if not os.environ.get("BORG_BASE_DIR"):
+        # Try to use XDG_CONFIG_HOME instead if BORG_BASE_DIR isn't explicitly set
-        config_home = os.environ.get("XDG_CONFIG_HOME", config_home)
+        if not os.environ.get("BORG_BASE_DIR"):
-    # Use BORG_CONFIG_DIR if set, otherwise assemble final path from config home path
+            config_home = os.environ.get("XDG_CONFIG_HOME", config_home)
-    config_dir = os.environ.get("BORG_CONFIG_DIR", os.path.join(config_home, "borg"))
+        # Use BORG_CONFIG_DIR if set, otherwise assemble final path from config home path
+        config_dir = os.environ.get("BORG_CONFIG_DIR", os.path.join(config_home, "borg"))
+    else:
+        config_dir = os.environ.get(
+            "BORG_CONFIG_DIR", join_base_dir(".config", legacy=legacy) or platformdirs.user_config_dir("borg")
+        )
+
     # Create path if it doesn't exist yet
     ensure_dir(config_dir)
     return config_dir

src/borg/testsuite/helpers.py

@@ -45,7 +45,7 @@ from ..helpers import eval_escapes
 from ..helpers import safe_unlink
 from ..helpers import text_to_json, binary_to_json
 from ..helpers.passphrase import Passphrase, PasswordRetriesExceeded
-from ..platform import is_cygwin
+from ..platform import is_cygwin, is_win32, is_darwin
 
 from . import BaseTestCase, FakeInputs, are_hardlinks_supported
 
@@ -584,60 +584,150 @@ def test_get_base_dir(monkeypatch):
     monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     monkeypatch.delenv("HOME", raising=False)
     monkeypatch.delenv("USER", raising=False)
-    assert get_base_dir() == os.path.expanduser("~")
+    assert get_base_dir(legacy=True) == os.path.expanduser("~")
     monkeypatch.setenv("USER", "root")
-    assert get_base_dir() == os.path.expanduser("~root")
+    assert get_base_dir(legacy=True) == os.path.expanduser("~root")
     monkeypatch.setenv("HOME", "/var/tmp/home")
-    assert get_base_dir() == "/var/tmp/home"
+    assert get_base_dir(legacy=True) == "/var/tmp/home"
     monkeypatch.setenv("BORG_BASE_DIR", "/var/tmp/base")
-    assert get_base_dir() == "/var/tmp/base"
+    assert get_base_dir(legacy=True) == "/var/tmp/base"
+    # non-legacy is much easier:
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
+    assert get_base_dir(legacy=False) is None
+    monkeypatch.setenv("BORG_BASE_DIR", "/var/tmp/base")
+    assert get_base_dir(legacy=False) == "/var/tmp/base"
+
+
+def test_get_base_dir_compat(monkeypatch):
+    """test that it works the same for legacy and for non-legacy implementation"""
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
+    # old way: if BORG_BASE_DIR is not set, make something up with HOME/USER/~
+    # new way: if BORG_BASE_DIR is not set, return None and let caller deal with it.
+    assert get_base_dir(legacy=False) is None
+    # new and old way: BORG_BASE_DIR overrides all other "base path determination".
+    monkeypatch.setenv("BORG_BASE_DIR", "/var/tmp/base")
+    assert get_base_dir(legacy=False) == get_base_dir(legacy=True)
 
 
 def test_get_config_dir(monkeypatch):
     """test that get_config_dir respects environment"""
-    monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    home_dir = os.path.expanduser("~")
-    assert get_config_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg")
+    if is_win32:
-    monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
-    assert get_config_dir() == os.path.join("/var/tmp/.config", "borg")
+        assert get_config_dir() == os.path.join(home_dir, "AppData", "Local", "borg", "borg")
-    monkeypatch.setenv("BORG_CONFIG_DIR", "/var/tmp")
+        monkeypatch.setenv("BORG_CONFIG_DIR", home_dir)
-    assert get_config_dir() == "/var/tmp"
+        assert get_config_dir() == home_dir
+    elif is_darwin:
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
+        assert get_config_dir() == os.path.join(home_dir, "Library", "Application Support", "borg")
+        monkeypatch.setenv("BORG_CONFIG_DIR", "/var/tmp")
+        assert get_config_dir() == "/var/tmp"
+    else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
+        assert get_config_dir() == os.path.join(home_dir, ".config", "borg")
+        monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+        assert get_config_dir() == os.path.join("/var/tmp/.config", "borg")
+        monkeypatch.setenv("BORG_CONFIG_DIR", "/var/tmp")
+        assert get_config_dir() == "/var/tmp"
+
+
+def test_get_config_dir_compat(monkeypatch):
+    """test that it works the same for legacy and for non-legacy implementation"""
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
+    if not is_darwin and not is_win32:
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        # fails on macOS: assert '/Users/tw/Library/Application Support/borg' == '/Users/tw/.config/borg'
+        # fails on win32 MSYS2 (but we do not need legacy compat there).
+        assert get_config_dir(legacy=False) == get_config_dir(legacy=True)
+    if not is_darwin and not is_win32:
+        monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config1")
+        # fails on macOS: assert '/Users/tw/Library/Application Support/borg' == '/var/tmp/.config1/borg'
+        # fails on win32 MSYS2 (but we do not need legacy compat there).
+        assert get_config_dir(legacy=False) == get_config_dir(legacy=True)
+    monkeypatch.setenv("BORG_CONFIG_DIR", "/var/tmp/.config2")
+    assert get_config_dir(legacy=False) == get_config_dir(legacy=True)
 
 
 def test_get_cache_dir(monkeypatch):
     """test that get_cache_dir respects environment"""
-    monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
-    monkeypatch.delenv("XDG_CACHE_HOME", raising=False)
+    home_dir = os.path.expanduser("~")
-    assert get_cache_dir() == os.path.join(os.path.expanduser("~"), ".cache", "borg")
+    if is_win32:
-    monkeypatch.setenv("XDG_CACHE_HOME", "/var/tmp/.cache")
+        monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
-    assert get_cache_dir() == os.path.join("/var/tmp/.cache", "borg")
+        assert get_cache_dir() == os.path.join(home_dir, "AppData", "Local", "borg", "borg", "Cache")
-    monkeypatch.setenv("BORG_CACHE_DIR", "/var/tmp")
+        monkeypatch.setenv("BORG_CACHE_DIR", home_dir)
-    assert get_cache_dir() == "/var/tmp"
+        assert get_cache_dir() == home_dir
+    elif is_darwin:
+        monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
+        assert get_cache_dir() == os.path.join(home_dir, "Library", "Caches", "borg")
+        monkeypatch.setenv("BORG_CACHE_DIR", "/var/tmp")
+        assert get_cache_dir() == "/var/tmp"
+    else:
+        monkeypatch.delenv("XDG_CACHE_HOME", raising=False)
+        monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
+        assert get_cache_dir() == os.path.join(home_dir, ".cache", "borg")
+        monkeypatch.setenv("XDG_CACHE_HOME", "/var/tmp/.cache")
+        assert get_cache_dir() == os.path.join("/var/tmp/.cache", "borg")
+        monkeypatch.setenv("BORG_CACHE_DIR", "/var/tmp")
+        assert get_cache_dir() == "/var/tmp"
 
 
 def test_get_keys_dir(monkeypatch):
     """test that get_keys_dir respects environment"""
-    monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    home_dir = os.path.expanduser("~")
-    assert get_keys_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg", "keys")
+    if is_win32:
-    monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+        monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
-    assert get_keys_dir() == os.path.join("/var/tmp/.config", "borg", "keys")
+        assert get_keys_dir() == os.path.join(home_dir, "AppData", "Local", "borg", "borg", "keys")
-    monkeypatch.setenv("BORG_KEYS_DIR", "/var/tmp")
+        monkeypatch.setenv("BORG_KEYS_DIR", home_dir)
-    assert get_keys_dir() == "/var/tmp"
+        assert get_keys_dir() == home_dir
+    elif is_darwin:
+        monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
+        assert get_keys_dir() == os.path.join(home_dir, "Library", "Application Support", "borg", "keys")
+        monkeypatch.setenv("BORG_KEYS_DIR", "/var/tmp")
+        assert get_keys_dir() == "/var/tmp"
+    else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
+        assert get_keys_dir() == os.path.join(home_dir, ".config", "borg", "keys")
+        monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+        assert get_keys_dir() == os.path.join("/var/tmp/.config", "borg", "keys")
+        monkeypatch.setenv("BORG_KEYS_DIR", "/var/tmp")
+        assert get_keys_dir() == "/var/tmp"
 
 
 def test_get_security_dir(monkeypatch):
     """test that get_security_dir respects environment"""
-    monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    home_dir = os.path.expanduser("~")
-    assert get_security_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg", "security")
+    if is_win32:
-    assert get_security_dir(repository_id="1234") == os.path.join(
+        monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
-        os.path.expanduser("~"), ".config", "borg", "security", "1234"
+        assert get_security_dir() == os.path.join(home_dir, "AppData", "Local", "borg", "borg", "security")
-    )
+        assert get_security_dir(repository_id="1234") == os.path.join(
-    monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+            home_dir, "AppData", "Local", "borg", "borg", "security", "1234"
-    assert get_security_dir() == os.path.join("/var/tmp/.config", "borg", "security")
+        )
-    monkeypatch.setenv("BORG_SECURITY_DIR", "/var/tmp")
+        monkeypatch.setenv("BORG_SECURITY_DIR", home_dir)
-    assert get_security_dir() == "/var/tmp"
+        assert get_security_dir() == home_dir
+    elif is_darwin:
+        monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
+        assert get_security_dir() == os.path.join(home_dir, "Library", "Application Support", "borg", "security")
+        assert get_security_dir(repository_id="1234") == os.path.join(
+            home_dir, "Library", "Application Support", "borg", "security", "1234"
+        )
+        monkeypatch.setenv("BORG_SECURITY_DIR", "/var/tmp")
+        assert get_security_dir() == "/var/tmp"
+    else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
+        assert get_security_dir() == os.path.join(home_dir, ".config", "borg", "security")
+        assert get_security_dir(repository_id="1234") == os.path.join(home_dir, ".config", "borg", "security", "1234")
+        monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
+        assert get_security_dir() == os.path.join("/var/tmp/.config", "borg", "security")
+        monkeypatch.setenv("BORG_SECURITY_DIR", "/var/tmp")
+        assert get_security_dir() == "/var/tmp"
 
 
 def test_file_size():