diff --git a/conftest.py b/conftest.py
index 5caf8a09a..312f57a20 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,7 @@
 import os
 
+import pytest
+
 from borg.logger import setup_logging
 
 # Ensure that the loggers exist for all tests
@@ -16,6 +18,17 @@ def pytest_configure(config):
     constants.PBKDF2_ITERATIONS = 1
 
 
+@pytest.fixture(autouse=True)
+def clean_env(tmpdir_factory, monkeypatch):
+    # avoid that we access / modify the user's normal .config / .cache directory:
+    monkeypatch.setenv('XDG_CONFIG_HOME', tmpdir_factory.mktemp('xdg-config-home'))
+    monkeypatch.setenv('XDG_CACHE_HOME', tmpdir_factory.mktemp('xdg-cache-home'))
+    # also avoid to use anything from the outside environment:
+    keys = [key for key in os.environ if key.startswith('BORG_')]
+    for key in keys:
+        monkeypatch.delenv(key, raising=False)
+
+
 def pytest_report_header(config, startdir):
     tests = {
         "BSD flags": has_lchflags,
diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py
index 55569e96e..bca3eb22d 100644
--- a/src/borg/testsuite/helpers.py
+++ b/src/borg/testsuite/helpers.py
@@ -605,44 +605,29 @@ def test(self):
         self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc))
 
 
-def test_get_cache_dir():
+def test_get_cache_dir(monkeypatch):
     """test that get_cache_dir respects environment"""
-    # reset BORG_CACHE_DIR in order to test default
-    old_env = None
-    if os.environ.get('BORG_CACHE_DIR'):
-        old_env = os.environ['BORG_CACHE_DIR']
-        del(os.environ['BORG_CACHE_DIR'])
+    monkeypatch.delenv('XDG_CACHE_HOME', raising=False)
     assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg')
-    os.environ['XDG_CACHE_HOME'] = '/var/tmp/.cache'
+    monkeypatch.setenv('XDG_CACHE_HOME', '/var/tmp/.cache')
     assert get_cache_dir() == os.path.join('/var/tmp/.cache', 'borg')
-    os.environ['BORG_CACHE_DIR'] = '/var/tmp'
+    monkeypatch.setenv('BORG_CACHE_DIR', '/var/tmp')
     assert get_cache_dir() == '/var/tmp'
-    # reset old env
-    if old_env is not None:
-        os.environ['BORG_CACHE_DIR'] = old_env
 
 
-def test_get_keys_dir():
+def test_get_keys_dir(monkeypatch):
     """test that get_keys_dir respects environment"""
-    # reset BORG_KEYS_DIR in order to test default
-    old_env = None
-    if os.environ.get('BORG_KEYS_DIR'):
-        old_env = os.environ['BORG_KEYS_DIR']
-        del(os.environ['BORG_KEYS_DIR'])
+    monkeypatch.delenv('XDG_CONFIG_HOME', raising=False)
     assert get_keys_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'keys')
-    os.environ['XDG_CONFIG_HOME'] = '/var/tmp/.config'
+    monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config')
     assert get_keys_dir() == os.path.join('/var/tmp/.config', 'borg', 'keys')
-    os.environ['BORG_KEYS_DIR'] = '/var/tmp'
+    monkeypatch.setenv('BORG_KEYS_DIR', '/var/tmp')
     assert get_keys_dir() == '/var/tmp'
-    # reset old env
-    if old_env is not None:
-        os.environ['BORG_KEYS_DIR'] = old_env
 
 
 def test_get_nonces_dir(monkeypatch):
     """test that get_nonces_dir respects environment"""
     monkeypatch.delenv('XDG_CONFIG_HOME', raising=False)
-    monkeypatch.delenv('BORG_NONCES_DIR', raising=False)
     assert get_nonces_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'key-nonces')
     monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config')
     assert get_nonces_dir() == os.path.join('/var/tmp/.config', 'borg', 'key-nonces')
diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py
index 94b455396..c2e637bd4 100644
--- a/src/borg/testsuite/key.py
+++ b/src/borg/testsuite/key.py
@@ -14,17 +14,6 @@
 from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex
 
 
-@pytest.fixture(autouse=True)
-def clean_env(monkeypatch):
-    # Workaround for some tests (testsuite/archiver) polluting the environment
-    monkeypatch.delenv('BORG_PASSPHRASE', False)
-
-
-@pytest.fixture(autouse=True)
-def nonce_dir(tmpdir_factory, monkeypatch):
-    monkeypatch.setenv('XDG_CONFIG_HOME', tmpdir_factory.mktemp('xdg-config-home'))
-
-
 class TestKey:
     class MockArgs:
         location = Location(tempfile.mkstemp()[1])
diff --git a/src/borg/testsuite/nonces.py b/src/borg/testsuite/nonces.py
index 88405f56d..14d1f52d5 100644
--- a/src/borg/testsuite/nonces.py
+++ b/src/borg/testsuite/nonces.py
@@ -10,17 +10,6 @@
 from .. import nonces  # for monkey patching NONCE_SPACE_RESERVATION
 
 
-@pytest.fixture(autouse=True)
-def clean_env(monkeypatch):
-    # Workaround for some tests (testsuite/archiver) polluting the environment
-    monkeypatch.delenv('BORG_PASSPHRASE', False)
-
-
-@pytest.fixture(autouse=True)
-def nonce_dir(tmpdir_factory, monkeypatch):
-    monkeypatch.setenv('XDG_CONFIG_HOME', tmpdir_factory.mktemp('xdg-config-home'))
-
-
 class TestNonceManager:
 
     class MockRepository: