Merge pull request #3060 from ThomasWaldmann/fix-spawn-passcommand-1.1

Fix subprocess environments (1.1-maint)

src/borg/archiver.py

@@ -64,7 +64,7 @@
 from .helpers import basic_json_data, json_print
 from .helpers import replace_placeholders
 from .helpers import ChunkIteratorFileWrapper
-from .helpers import popen_with_error_handling
+from .helpers import popen_with_error_handling, prepare_subprocess_env
 from .helpers import dash_open
 from .helpers import umount
 from .nanorst import rst_to_terminal
@@ -790,10 +790,12 @@ def do_export_tar(self, args, repository, manifest, key, archive):
             # The decision whether to close that or not remains the same.
             filterout = tarstream
             filterout_close = tarstream_close
+            env = prepare_subprocess_env(system=True)
             # There is no deadlock potential here (the subprocess docs warn about this), because
             # communication with the process is a one-way road, i.e. the process can never block
             # for us to do something while we block on the process for something different.
-            filterproc = popen_with_error_handling(filter, stdin=subprocess.PIPE, stdout=filterout, log_prefix='--tar-filter: ')
+            filterproc = popen_with_error_handling(filter, stdin=subprocess.PIPE, stdout=filterout,
+                                                   log_prefix='--tar-filter: ', env=env)
             if not filterproc:
                 return EXIT_ERROR
             # Always close the pipe, otherwise the filter process would not notice when we are done.
@@ -1677,9 +1679,10 @@ def do_with_lock(self, args, repository):
         # we can only do this for local repositories (with .io), though:
         if hasattr(repository, 'io'):
             repository.io.close_segment()
+        env = prepare_subprocess_env(system=True)
         try:
             # we exit with the return code we get from the subprocess
-            return subprocess.call([args.command] + args.args)
+            return subprocess.call([args.command] + args.args, env=env)
         finally:
             # we need to commit the "no change" operation we did to the manifest
             # because it created a new segment file in the repository. if we would

src/borg/crypto/key.py

@@ -23,6 +23,7 @@
 from ..helpers import get_keys_dir, get_security_dir
 from ..helpers import get_limited_unpacker
 from ..helpers import bin_to_hex
+from ..helpers import prepare_subprocess_env
 from ..item import Key, EncryptedKey
 from ..platform import SaveFile
 from .nonces import NonceManager
@@ -431,8 +432,10 @@ def env_passphrase(cls, default=None):
     def env_passcommand(cls, default=None):
         passcommand = os.environ.get('BORG_PASSCOMMAND', None)
         if passcommand is not None:
+            # passcommand is a system command (not inside pyinstaller env)
+            env = prepare_subprocess_env(system=True)
             try:
-                passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True)
+                passphrase = subprocess.check_output(shlex.split(passcommand), universal_newlines=True, env=env)
             except (subprocess.CalledProcessError, FileNotFoundError) as e:
                 raise PasscommandFailure(e)
             return cls(passphrase.rstrip('\n'))

src/borg/helpers.py

@@ -2303,6 +2303,32 @@ def popen_with_error_handling(cmd_line: str, log_prefix='', **kwargs):
         return
 
 
+def prepare_subprocess_env(system, env=None):
+    """
+    Prepare the environment for a subprocess we are going to create.
+
+    :param system: True for preparing to invoke system-installed binaries,
+                   False for stuff inside the pyinstaller environment (like borg, python).
+    :param env: optionally give a environment dict here. if not given, default to os.environ.
+    :return: a modified copy of the environment
+    """
+    env = dict(env if env is not None else os.environ)
+    if system:
+        # a pyinstaller binary's bootloader modifies LD_LIBRARY_PATH=/tmp/_ME...,
+        # but we do not want that system binaries (like ssh or other) pick up
+        # (non-matching) libraries from there.
+        # thus we install the original LDLP, before pyinstaller has modified it:
+        lp_key = 'LD_LIBRARY_PATH'
+        lp_orig = env.get(lp_key + '_ORIG')  # pyinstaller >= 20160820 has this
+        if lp_orig is not None:
+            env[lp_key] = lp_orig
+    # security: do not give secrets to subprocess
+    env.pop('BORG_PASSPHRASE', None)
+    # for information, give borg version to the subprocess
+    env['BORG_VERSION'] = borg_version
+    return env
+
+
 def dash_open(path, mode):
     assert '+' not in mode  # the streams are either r or w, but never both
     if path == '-':
@@ -2317,7 +2343,8 @@ def is_terminal(fd=sys.stdout):
 
 
 def umount(mountpoint):
+    env = prepare_subprocess_env(system=True)
     try:
-        return subprocess.call(['fusermount', '-u', mountpoint])
+        return subprocess.call(['fusermount', '-u', mountpoint], env=env)
     except FileNotFoundError:
-        return subprocess.call(['umount', mountpoint])
+        return subprocess.call(['umount', mountpoint], env=env)

src/borg/remote.py

@@ -30,6 +30,7 @@
 from .helpers import sysinfo
 from .helpers import format_file_size
 from .helpers import truncate_and_unlink
+from .helpers import prepare_subprocess_env
 from .logger import create_logger, setup_logging
 from .repository import Repository
 from .version import parse_version, format_version
@@ -537,21 +538,12 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock
         self.server_version = parse_version('1.0.8')  # fallback version if server is too old to send version information
         self.p = None
         testing = location.host == '__testsuite__'
+        # when testing, we invoke and talk to a borg process directly (no ssh).
+        # when not testing, we invoke the system-installed ssh binary to talk to a remote borg.
+        env = prepare_subprocess_env(system=not testing)
         borg_cmd = self.borg_cmd(args, testing)
-        env = dict(os.environ)
         if not testing:
             borg_cmd = self.ssh_cmd(location) + borg_cmd
-            # pyinstaller binary modifies LD_LIBRARY_PATH=/tmp/_ME... but we do not want
-            # that the system's ssh binary picks up (non-matching) libraries from there.
-            # thus we install the original LDLP, before pyinstaller has modified it:
-            lp_key = 'LD_LIBRARY_PATH'
-            lp_orig = env.get(lp_key + '_ORIG')  # pyinstaller >= 20160820 has this
-            if lp_orig is not None:
-                env[lp_key] = lp_orig
-            else:
-                env.pop(lp_key, None)
-        env.pop('BORG_PASSPHRASE', None)  # security: do not give secrets to subprocess
-        env['BORG_VERSION'] = __version__
         logger.debug('SSH command line: %s', borg_cmd)
         self.p = Popen(borg_cmd, bufsize=0, stdin=PIPE, stdout=PIPE, stderr=PIPE, env=env)
         self.stdin_fd = self.p.stdin.fileno()

src/borg/xattr.py

@@ -10,7 +10,7 @@
 from ctypes.util import find_library
 from distutils.version import LooseVersion
 
-from .helpers import Buffer
+from .helpers import Buffer, prepare_subprocess_env
 
 
 try:
@@ -88,7 +88,9 @@ def get_all(path, follow_symlinks=True):
     preloads = re.split("[ :]", LD_PRELOAD)
     for preload in preloads:
         if preload.startswith("libfakeroot"):
-            fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1])
+            env = prepare_subprocess_env(system=True)
+            fakeroot_output = subprocess.check_output(['fakeroot', '-v'], env=env)
+            fakeroot_version = LooseVersion(fakeroot_output.decode('ascii').split()[-1])
             if fakeroot_version >= LooseVersion("1.20.2"):
                 # 1.20.2 has been confirmed to have xattr support
                 # 1.18.2 has been confirmed not to have xattr support