diff --git a/conftest.py b/conftest.py
index 1c08cbbda..362ee0de5 100644
--- a/conftest.py
+++ b/conftest.py
@@ -20,13 +20,12 @@
 
 @pytest.fixture(autouse=True)
 def clean_env(tmpdir_factory, monkeypatch):
-    # avoid that we access / modify the user's normal .config / .cache directory:
-    monkeypatch.setenv("XDG_CONFIG_HOME", str(tmpdir_factory.mktemp("xdg-config-home")))
-    monkeypatch.setenv("XDG_CACHE_HOME", str(tmpdir_factory.mktemp("xdg-cache-home")))
     # also avoid to use anything from the outside environment:
     keys = [key for key in os.environ if key.startswith("BORG_") and key not in ("BORG_FUSE_IMPL",)]
     for key in keys:
         monkeypatch.delenv(key, raising=False)
+    # avoid that we access / modify the user's normal .config / .cache directory:
+    monkeypatch.setenv("BORG_BASE_DIR", str(tmpdir_factory.mktemp("borg-base-dir")))
     # Speed up tests
     monkeypatch.setenv("BORG_TESTONLY_WEAKEN_KDF", "1")
 
diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py
index fffe7652a..01e9f4167 100644
--- a/src/borg/testsuite/helpers.py
+++ b/src/borg/testsuite/helpers.py
@@ -606,11 +606,13 @@ def test_get_base_dir_compat(monkeypatch):
 
 def test_get_config_dir(monkeypatch):
     """test that get_config_dir respects environment"""
-    monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     if is_win32:
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
         assert get_config_dir(legacy=False) == os.path.join(os.path.expanduser("~"), "AppData", "Local", "borg", "borg")
     else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
         assert get_config_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg")
         monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
         assert get_config_dir() == os.path.join("/var/tmp/.config", "borg")
@@ -620,6 +622,7 @@ def test_get_config_dir(monkeypatch):
 
 def test_get_config_dir_compat(monkeypatch):
     """test that it works the same for legacy and for non-legacy implementation"""
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     if not is_darwin:
         monkeypatch.delenv("BORG_CONFIG_DIR", raising=False)
         monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
@@ -635,13 +638,15 @@ def test_get_config_dir_compat(monkeypatch):
 
 def test_get_cache_dir(monkeypatch):
     """test that get_cache_dir respects environment"""
-    monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
-    monkeypatch.delenv("XDG_CACHE_HOME", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     if is_win32:
+        monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
         assert get_cache_dir(legacy=False) == os.path.join(
             os.path.expanduser("~"), "AppData", "Local", "borg", "borg", "Cache"
         )
     else:
+        monkeypatch.delenv("XDG_CACHE_HOME", raising=False)
+        monkeypatch.delenv("BORG_CACHE_DIR", raising=False)
         assert get_cache_dir() == os.path.join(os.path.expanduser("~"), ".cache", "borg")
         monkeypatch.setenv("XDG_CACHE_HOME", "/var/tmp/.cache")
         assert get_cache_dir() == os.path.join("/var/tmp/.cache", "borg")
@@ -651,13 +656,15 @@ def test_get_cache_dir(monkeypatch):
 
 def test_get_keys_dir(monkeypatch):
     """test that get_keys_dir respects environment"""
-    monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     if is_win32:
+        monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
         assert get_keys_dir(legacy=False) == os.path.join(
             os.path.expanduser("~"), "AppData", "Local", "borg", "borg", "keys"
         )
     else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_KEYS_DIR", raising=False)
         assert get_keys_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg", "keys")
         monkeypatch.setenv("XDG_CONFIG_HOME", "/var/tmp/.config")
         assert get_keys_dir() == os.path.join("/var/tmp/.config", "borg", "keys")
@@ -667,9 +674,9 @@ def test_get_keys_dir(monkeypatch):
 
 def test_get_security_dir(monkeypatch):
     """test that get_security_dir respects environment"""
-    monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
-    monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+    monkeypatch.delenv("BORG_BASE_DIR", raising=False)
     if is_win32:
+        monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
         assert get_security_dir(legacy=False) == os.path.join(
             os.path.expanduser("~"), "AppData", "Local", "borg", "borg", "security"
         )
@@ -677,6 +684,8 @@ def test_get_security_dir(monkeypatch):
             os.path.expanduser("~"), "AppData", "Local", "borg", "borg", "security", "1234"
         )
     else:
+        monkeypatch.delenv("XDG_CONFIG_HOME", raising=False)
+        monkeypatch.delenv("BORG_SECURITY_DIR", raising=False)
         assert get_security_dir() == os.path.join(os.path.expanduser("~"), ".config", "borg", "security")
         assert get_security_dir(repository_id="1234") == os.path.join(
             os.path.expanduser("~"), ".config", "borg", "security", "1234"