Merge pull request #4317 from ThomasWaldmann/workaround-wsl-sync_file_range-issue

work around some Microsoft WSL issues

src/borg/helpers.py

@@ -139,7 +139,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if borg.crypto.low_level.API_VERSION != '1.1_02':
         raise ExtensionModuleError
-    if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_03':
+    if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_04':
         raise ExtensionModuleError
     if item.API_VERSION != '1.1_02':
         raise ExtensionModuleError

src/borg/platform/base.py

@@ -17,7 +17,7 @@
 are correctly composed into the base functionality.
 """
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.1_04'
 
 fdatasync = getattr(os, 'fdatasync', os.fsync)
 

src/borg/platform/darwin.pyx

@@ -6,7 +6,7 @@ from ..helpers import user2uid, group2gid
 from ..helpers import safe_decode, safe_encode
 from .posix import swidth
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.1_04'
 
 cdef extern from "sys/acl.h":
     ctypedef struct _acl_t:

src/borg/platform/freebsd.pyx

@@ -6,7 +6,7 @@ from ..helpers import posix_acl_use_stored_uid_gid
 from ..helpers import safe_encode, safe_decode
 from .posix import swidth
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.1_04'
 
 cdef extern from "errno.h":
     int errno

src/borg/platform/linux.pyx

@@ -15,7 +15,7 @@ from .posix import swidth
 from libc cimport errno
 from libc.stdint cimport int64_t
 
-API_VERSION = '1.1_03'
+API_VERSION = '1.1_04'
 
 cdef extern from "sys/types.h":
     int ACL_TYPE_ACCESS
@@ -228,6 +228,7 @@ def acl_set(path, item, numeric_owner=False):
         finally:
             acl_free(default_acl)
 
+
 cdef _sync_file_range(fd, offset, length, flags):
     assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset
     assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
@@ -235,39 +236,60 @@ cdef _sync_file_range(fd, offset, length, flags):
         raise OSError(errno.errno, os.strerror(errno.errno))
     safe_fadvise(fd, offset, length, 'DONTNEED')
 
+
 cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1
 
 
-class SyncFile(BaseSyncFile):
+def _is_WSL():
-    """
+    """detect Windows Subsystem for Linux"""
-    Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.
+    try:
+        with open('/proc/version') as fd:
+            linux_version = fd.read()
+        # hopefully no non-WSL Linux will ever mention 'Microsoft' in the kernel version:
+        return 'Microsoft' in linux_version
+    except:  # noqa
+        # make sure to never ever crash due to this check.
+        return False
 
-    "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
-    disk in the immediate future.
-    """
 
-    def __init__(self, path, binary=False):
+if _is_WSL():
-        super().__init__(path, binary)
+    class SyncFile(BaseSyncFile):
-        self.offset = 0
+        # if we are on Microsoft's "Windows Subsytem for Linux", use the
-        self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
+        # more generic BaseSyncFile to avoid issues like seen there:
-        self.last_sync = 0
+        # https://github.com/borgbackup/borg/issues/1961
-        self.pending_sync = None
+        pass
+else:
+    # a real Linux, so we can do better. :)
+    class SyncFile(BaseSyncFile):
+        """
+        Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.
 
-    def write(self, data):
+        "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
-        self.offset += self.fd.write(data)
+        disk in the immediate future.
-        offset = self.offset & ~PAGE_MASK
+        """
-        if offset >= self.last_sync + self.write_window:
+
+        def __init__(self, path, binary=False):
+            super().__init__(path, binary)
+            self.offset = 0
+            self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
+            self.last_sync = 0
+            self.pending_sync = None
+
+        def write(self, data):
+            self.offset += self.fd.write(data)
+            offset = self.offset & ~PAGE_MASK
+            if offset >= self.last_sync + self.write_window:
+                self.fd.flush()
+                _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
+                if self.pending_sync is not None:
+                    _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
+                                     SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
+                self.pending_sync = self.last_sync
+                self.last_sync = offset
+
+        def sync(self):
             self.fd.flush()
-            _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
+            os.fdatasync(self.fileno)
-            if self.pending_sync is not None:
+            # tell the OS that it does not need to cache what we just wrote,
-                _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
+            # avoids spoiling the cache for the OS and other processes.
-                                 SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
+            safe_fadvise(self.fileno, 0, 0, 'DONTNEED')
-            self.pending_sync = self.last_sync
-            self.last_sync = offset
-
-    def sync(self):
-        self.fd.flush()
-        os.fdatasync(self.fileno)
-        # tell the OS that it does not need to cache what we just wrote,
-        # avoids spoiling the cache for the OS and other processes.
-        safe_fadvise(self.fileno, 0, 0, 'DONTNEED')