Merge pull request #2528 from ThomasWaldmann/follow-symlinks

document follow_symlinks requirements, check libc, fixes #2507

docs/installation.rst

@@ -54,6 +54,21 @@ manages data.
 
 .. _locking: https://en.wikipedia.org/wiki/File_locking#Lock_files
 
+(G)LIBC requirements
+--------------------
+
+Borg uses some filesytem functions from Python's `os` standard library module
+with `follow_symlinks=False`. These are implemented since quite a while with
+the non-symlink-following (g)libc functions like e.g. `lstat` or `lutimes`
+(not: `stat` or `utimes`).
+
+Some stoneage systems (like RHEL/CentOS 5) and also Python interpreter binaries
+compiled to be able to run on such systems (like Python installed via Anaconda)
+might miss these functions and Borg won't be able to work correctly.
+This issue will be detected early and Borg will abort with a fatal error.
+
+For the Borg binaries, there are additional (g)libc requirements, see below.
+
 .. _distribution-package:
 
 Distribution Package

src/borg/archive.py

@@ -569,7 +569,7 @@ Utilization of max. archive size: {csize_max:.0%}
         path = os.path.join(dest, item.path)
         # Attempt to remove existing files, ignore errors on failure
         try:
-            st = os.lstat(path)
+            st = os.stat(path, follow_symlinks=False)
             if stat.S_ISDIR(st.st_mode):
                 os.rmdir(path)
             else:
@@ -674,7 +674,7 @@ Utilization of max. archive size: {csize_max:.0%}
             if fd:
                 os.fchown(fd, uid, gid)
             else:
-                os.lchown(path, uid, gid)
+                os.chown(path, uid, gid, follow_symlinks=False)
         except OSError:
             pass
         if fd:

src/borg/archiver.py

@@ -54,7 +54,7 @@ from .helpers import get_cache_dir
 from .helpers import Manifest
 from .helpers import hardlinkable
 from .helpers import StableDict
-from .helpers import check_extension_modules
+from .helpers import check_python, check_extension_modules
 from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
 from .helpers import log_multi
 from .helpers import signal_handler, raising_signal_handler, SigHup, SigTerm
@@ -436,7 +436,7 @@ class Archiver:
                     continue
                 path = os.path.normpath(path)
                 try:
-                    st = os.lstat(path)
+                    st = os.stat(path, follow_symlinks=False)
                 except OSError as e:
                     self.print_warning('%s: %s', path, e)
                     continue
@@ -498,7 +498,7 @@ class Archiver:
         """
         if st is None:
             with backup_io('stat'):
-                st = os.lstat(path)
+                st = os.stat(path, follow_symlinks=False)
 
         recurse_excluded_dir = False
         if not matcher.match(path):
@@ -3829,6 +3829,7 @@ class Archiver:
         return args
 
     def prerun_checks(self, logger):
+        check_python()
         check_extension_modules()
         selftest(logger)
 

src/borg/helpers.py

@@ -114,6 +114,16 @@ class InvalidPlaceholder(PlaceholderError):
     """Invalid placeholder "{}" in string: {}"""
 
 
+class PythonLibcTooOld(Error):
+    """FATAL: this Python was compiled for a too old (g)libc and misses required functionality."""
+
+
+def check_python():
+    required_funcs = {os.stat, os.utime, os.chown}
+    if not os.supports_follow_symlinks.issuperset(required_funcs):
+        raise PythonLibcTooOld
+
+
 def check_extension_modules():
     from . import platform, compress, item
     if hashindex.API_VERSION != '1.1_01':
@@ -1754,7 +1764,7 @@ class GenericDirEntry:
 
     def stat(self, follow_symlinks=True):
         assert not follow_symlinks
-        return os.lstat(self.path)
+        return os.stat(self.path, follow_symlinks=follow_symlinks)
 
     def _check_type(self, type):
         st = self.stat(False)

src/borg/testsuite/__init__.py

@@ -30,13 +30,11 @@ except ImportError:
     raises = None
 
 has_lchflags = hasattr(os, 'lchflags') or sys.platform.startswith('linux')
-no_lchlfags_because = '' if has_lchflags else '(not supported on this platform)'
 try:
     with tempfile.NamedTemporaryFile() as file:
         platform.set_flags(file.name, stat.UF_NODUMP)
 except OSError:
     has_lchflags = False
-    no_lchlfags_because = '(the file system at %s does not support flags)' % tempfile.gettempdir()
 
 try:
     import llfuse
@@ -67,7 +65,7 @@ def are_symlinks_supported():
     with unopened_tempfile() as filepath:
         try:
             os.symlink('somewhere', filepath)
-            if os.lstat(filepath) and os.readlink(filepath) == 'somewhere':
+            if os.stat(filepath, follow_symlinks=False) and os.readlink(filepath) == 'somewhere':
                 return True
         except OSError:
             pass
@@ -109,7 +107,7 @@ def is_utime_fully_supported():
             open(filepath, 'w').close()
         try:
             os.utime(filepath, (1000, 2000), follow_symlinks=False)
-            new_stats = os.lstat(filepath)
+            new_stats = os.stat(filepath, follow_symlinks=False)
             if new_stats.st_atime == 1000 and new_stats.st_mtime == 2000:
                 return True
         except OSError as err:
@@ -158,8 +156,8 @@ class BaseTestCase(unittest.TestCase):
         for filename in diff.common:
             path1 = os.path.join(diff.left, filename)
             path2 = os.path.join(diff.right, filename)
-            s1 = os.lstat(path1)
+            s1 = os.stat(path1, follow_symlinks=False)
-            s2 = os.lstat(path2)
+            s2 = os.stat(path2, follow_symlinks=False)
             # Assume path2 is on FUSE if st_dev is different
             fuse = s1.st_dev != s2.st_dev
             attrs = ['st_uid', 'st_gid', 'st_rdev']