From 53649cd6e69257e08af5109ed8ba9413902422ea Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Feb 2019 01:30:54 +0100 Subject: [PATCH] work around some Microsoft WSL issues see there: https://github.com/borgbackup/borg/issues/1961 and especially there (not implemented sync_file_range): https://github.com/Microsoft/WSL/issues/645 --- src/borg/helpers.py | 2 +- src/borg/platform/base.py | 2 +- src/borg/platform/darwin.pyx | 2 +- src/borg/platform/freebsd.pyx | 2 +- src/borg/platform/linux.pyx | 82 ++++++++++++++++++++++------------- 5 files changed, 56 insertions(+), 34 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 31946919c..ed6b3afeb 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -139,7 +139,7 @@ def check_extension_modules(): raise ExtensionModuleError if borg.crypto.low_level.API_VERSION != '1.1_02': raise ExtensionModuleError - if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_03': + if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_04': raise ExtensionModuleError if item.API_VERSION != '1.1_02': raise ExtensionModuleError diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index 96a26e3ac..b046b6597 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -17,7 +17,7 @@ platform API: that way platform APIs provided by the platform-specific support m are correctly composed into the base functionality. """ -API_VERSION = '1.1_03' +API_VERSION = '1.1_04' fdatasync = getattr(os, 'fdatasync', os.fsync) diff --git a/src/borg/platform/darwin.pyx b/src/borg/platform/darwin.pyx index f8bdd17a5..6057fec29 100644 --- a/src/borg/platform/darwin.pyx +++ b/src/borg/platform/darwin.pyx @@ -6,7 +6,7 @@ from ..helpers import user2uid, group2gid from ..helpers import safe_decode, safe_encode from .posix import swidth -API_VERSION = '1.1_03' +API_VERSION = '1.1_04' cdef extern from "sys/acl.h": ctypedef struct _acl_t: diff --git a/src/borg/platform/freebsd.pyx b/src/borg/platform/freebsd.pyx index 2797da309..77f88f7f7 100644 --- a/src/borg/platform/freebsd.pyx +++ b/src/borg/platform/freebsd.pyx @@ -6,7 +6,7 @@ from ..helpers import posix_acl_use_stored_uid_gid from ..helpers import safe_encode, safe_decode from .posix import swidth -API_VERSION = '1.1_03' +API_VERSION = '1.1_04' cdef extern from "errno.h": int errno diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index d800bdfb1..6f1488c0d 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -15,7 +15,7 @@ from .posix import swidth from libc cimport errno from libc.stdint cimport int64_t -API_VERSION = '1.1_03' +API_VERSION = '1.1_04' cdef extern from "sys/types.h": int ACL_TYPE_ACCESS @@ -228,6 +228,7 @@ def acl_set(path, item, numeric_owner=False): finally: acl_free(default_acl) + cdef _sync_file_range(fd, offset, length, flags): assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset assert length & PAGE_MASK == 0, "length %d not page-aligned" % length @@ -235,39 +236,60 @@ cdef _sync_file_range(fd, offset, length, flags): raise OSError(errno.errno, os.strerror(errno.errno)) safe_fadvise(fd, offset, length, 'DONTNEED') + cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1 -class SyncFile(BaseSyncFile): - """ - Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability. +def _is_WSL(): + """detect Windows Subsystem for Linux""" + try: + with open('/proc/version') as fd: + linux_version = fd.read() + # hopefully no non-WSL Linux will ever mention 'Microsoft' in the kernel version: + return 'Microsoft' in linux_version + except: # noqa + # make sure to never ever crash due to this check. + return False - "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to - disk in the immediate future. - """ - def __init__(self, path, binary=False): - super().__init__(path, binary) - self.offset = 0 - self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK - self.last_sync = 0 - self.pending_sync = None +if _is_WSL(): + class SyncFile(BaseSyncFile): + # if we are on Microsoft's "Windows Subsytem for Linux", use the + # more generic BaseSyncFile to avoid issues like seen there: + # https://github.com/borgbackup/borg/issues/1961 + pass +else: + # a real Linux, so we can do better. :) + class SyncFile(BaseSyncFile): + """ + Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability. - def write(self, data): - self.offset += self.fd.write(data) - offset = self.offset & ~PAGE_MASK - if offset >= self.last_sync + self.write_window: + "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to + disk in the immediate future. + """ + + def __init__(self, path, binary=False): + super().__init__(path, binary) + self.offset = 0 + self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK + self.last_sync = 0 + self.pending_sync = None + + def write(self, data): + self.offset += self.fd.write(data) + offset = self.offset & ~PAGE_MASK + if offset >= self.last_sync + self.write_window: + self.fd.flush() + _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE) + if self.pending_sync is not None: + _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync, + SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER) + self.pending_sync = self.last_sync + self.last_sync = offset + + def sync(self): self.fd.flush() - _sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE) - if self.pending_sync is not None: - _sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync, - SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER) - self.pending_sync = self.last_sync - self.last_sync = offset - - def sync(self): - self.fd.flush() - os.fdatasync(self.fileno) - # tell the OS that it does not need to cache what we just wrote, - # avoids spoiling the cache for the OS and other processes. - safe_fadvise(self.fileno, 0, 0, 'DONTNEED') + os.fdatasync(self.fileno) + # tell the OS that it does not need to cache what we just wrote, + # avoids spoiling the cache for the OS and other processes. + safe_fadvise(self.fileno, 0, 0, 'DONTNEED')