Merge pull request #2722 from ThomasWaldmann/backports5

even more backports

borg/_chunker.c

@@ -39,7 +39,7 @@ static uint32_t table_base[] =
     0xc5ae37bb, 0xa76ce12a, 0x8150d8f3, 0x2ec29218, 0xa35f0984, 0x48c0647e, 0x0b5ff98c, 0x71893f7b
 };
 
-#define BARREL_SHIFT(v, shift) ( ((v) << shift) | ((v) >> (32 - shift)) )
+#define BARREL_SHIFT(v, shift) ( ((v) << shift) | ((v) >> ((32 - shift) & 0x1f)) )
 
 size_t pagemask;
 

borg/archive.py

@@ -381,7 +381,7 @@ Number of files: {0.stats.nfiles}'''.format(
         path = os.path.join(dest, item[b'path'])
         # Attempt to remove existing files, ignore errors on failure
         try:
-            st = os.lstat(path)
+            st = os.stat(path, follow_symlinks=False)
             if stat.S_ISDIR(st.st_mode):
                 os.rmdir(path)
             else:
@@ -472,7 +472,7 @@ Number of files: {0.stats.nfiles}'''.format(
             if fd:
                 os.fchown(fd, uid, gid)
             else:
-                os.lchown(path, uid, gid)
+                os.chown(path, uid, gid, follow_symlinks=False)
         except OSError:
             pass
         if fd:

borg/archiver.py

@@ -21,7 +21,7 @@ import collections
 from . import __version__
 from .helpers import Error, location_validator, archivename_validator, format_line, format_time, format_file_size, \
     parse_pattern, PathPrefixPattern, to_localtime, timestamp, safe_timestamp, bin_to_hex, \
-    get_cache_dir, prune_within, prune_split, \
+    get_cache_dir, prune_within, prune_split, check_python, \
     Manifest, NoManifestError, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \
     dir_is_tagged, bigint_to_int, ChunkerParams, CompressionSpec, PrefixSpec, is_slow_msgpack, yes, sysinfo, \
     EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, log_multi, PatternMatcher, ErrorIgnoringTextIOWrapper, set_ec, \
@@ -293,7 +293,7 @@ class Archiver:
                 path = os.path.normpath(path)
                 if args.one_file_system:
                     try:
-                        restrict_dev = os.lstat(path).st_dev
+                        restrict_dev = os.stat(path, follow_symlinks=False).st_dev
                     except OSError as e:
                         self.print_warning('%s: %s', path, e)
                         continue
@@ -346,7 +346,7 @@ class Archiver:
 
         try:
             with backup_io():
-                st = os.lstat(path)
+                st = os.stat(path, follow_symlinks=False)
             if (st.st_ino, st.st_dev) in skip_inodes:
                 return
             # Entering a new filesystem?
@@ -2079,13 +2079,17 @@ class Archiver:
         update_excludes(args)
         return args
 
+    def prerun_checks(self, logger):
+        check_python()
+        check_extension_modules()
+
     def run(self, args):
         os.umask(args.umask)  # early, before opening files
         self.lock_wait = args.lock_wait
         # This works around http://bugs.python.org/issue9351
         func = getattr(args, 'func', None) or getattr(args, 'fallback_func')
         setup_logging(level=args.log_level, is_serve=func == self.do_serve)  # do not use loggers before this!
-        check_extension_modules()
+        self.prerun_checks(logger)
         if is_slow_msgpack():
             logger.warning("Using a pure-python msgpack! This will result in lower performance.")
         return set_ec(func(args))

borg/chunker.pyx

@@ -48,11 +48,11 @@ cdef class Chunker:
         return chunker_process(self.chunker)
 
 
-def buzhash(unsigned char *data, unsigned long seed):
+def buzhash(data, unsigned long seed):
     cdef uint32_t *table
     cdef uint32_t sum
     table = buzhash_init_table(seed & 0xffffffff)
-    sum = c_buzhash(data, len(data), table)
+    sum = c_buzhash(<const unsigned char *> data, len(data), table)
     free(table)
     return sum
 

borg/helpers.py

@@ -115,6 +115,16 @@ class MandatoryFeatureUnsupported(Error):
     """Unsupported repository feature(s) {}. A newer version of borg is required to access this repository."""
 
 
+class PythonLibcTooOld(Error):
+    """FATAL: this Python was compiled for a too old (g)libc and misses required functionality."""
+
+
+def check_python():
+    required_funcs = {os.stat, os.utime, os.chown}
+    if not os.supports_follow_symlinks.issuperset(required_funcs):
+        raise PythonLibcTooOld
+
+
 def check_extension_modules():
     from . import platform, compress
     if hashindex.API_VERSION != '1.0_01':

borg/testsuite/__init__.py

@@ -73,8 +73,8 @@ class BaseTestCase(unittest.TestCase):
         for filename in diff.common:
             path1 = os.path.join(diff.left, filename)
             path2 = os.path.join(diff.right, filename)
-            s1 = os.lstat(path1)
+            s1 = os.stat(path1, follow_symlinks=False)
-            s2 = os.lstat(path2)
+            s2 = os.stat(path2, follow_symlinks=False)
             # Assume path2 is on FUSE if st_dev is different
             fuse = s1.st_dev != s2.st_dev
             attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev']

docs/installation.rst

@@ -20,6 +20,26 @@ There are different ways to install |project_name|:
     have the latest code or use revision control (each release is
     tagged).
 
+.. _installation-requirements:
+
+Pre-Installation Considerations
+-------------------------------
+
+(G)LIBC requirements
+--------------------
+
+Borg uses some filesytem functions from Python's `os` standard library module
+with `follow_symlinks=False`. These are implemented since quite a while with
+the non-symlink-following (g)libc functions like e.g. `lstat` or `lutimes`
+(not: `stat` or `utimes`).
+
+Some stoneage systems (like RHEL/CentOS 5) and also Python interpreter binaries
+compiled to be able to run on such systems (like Python installed via Anaconda)
+might miss these functions and Borg won't be able to work correctly.
+This issue will be detected early and Borg will abort with a fatal error.
+
+For the Borg binaries, there are additional (g)libc requirements, see below.
+
 .. _distribution-package:
 
 Distribution Package