work around some Microsoft WSL issues

see there:

https://github.com/borgbackup/borg/issues/1961

and especially there (not implemented sync_file_range):

https://github.com/Microsoft/WSL/issues/645
This commit is contained in:
Thomas Waldmann 2019-02-05 01:30:54 +01:00
parent ee6665964e
commit 53649cd6e6
5 changed files with 56 additions and 34 deletions

View File

@ -139,7 +139,7 @@ def check_extension_modules():
raise ExtensionModuleError
if borg.crypto.low_level.API_VERSION != '1.1_02':
raise ExtensionModuleError
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_03':
if platform.API_VERSION != platform.OS_API_VERSION or platform.API_VERSION != '1.1_04':
raise ExtensionModuleError
if item.API_VERSION != '1.1_02':
raise ExtensionModuleError

View File

@ -17,7 +17,7 @@ platform API: that way platform APIs provided by the platform-specific support m
are correctly composed into the base functionality.
"""
API_VERSION = '1.1_03'
API_VERSION = '1.1_04'
fdatasync = getattr(os, 'fdatasync', os.fsync)

View File

@ -6,7 +6,7 @@ from ..helpers import user2uid, group2gid
from ..helpers import safe_decode, safe_encode
from .posix import swidth
API_VERSION = '1.1_03'
API_VERSION = '1.1_04'
cdef extern from "sys/acl.h":
ctypedef struct _acl_t:

View File

@ -6,7 +6,7 @@ from ..helpers import posix_acl_use_stored_uid_gid
from ..helpers import safe_encode, safe_decode
from .posix import swidth
API_VERSION = '1.1_03'
API_VERSION = '1.1_04'
cdef extern from "errno.h":
int errno

View File

@ -15,7 +15,7 @@ from .posix import swidth
from libc cimport errno
from libc.stdint cimport int64_t
API_VERSION = '1.1_03'
API_VERSION = '1.1_04'
cdef extern from "sys/types.h":
int ACL_TYPE_ACCESS
@ -228,6 +228,7 @@ def acl_set(path, item, numeric_owner=False):
finally:
acl_free(default_acl)
cdef _sync_file_range(fd, offset, length, flags):
assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset
assert length & PAGE_MASK == 0, "length %d not page-aligned" % length
@ -235,39 +236,60 @@ cdef _sync_file_range(fd, offset, length, flags):
raise OSError(errno.errno, os.strerror(errno.errno))
safe_fadvise(fd, offset, length, 'DONTNEED')
cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1
class SyncFile(BaseSyncFile):
"""
Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.
def _is_WSL():
"""detect Windows Subsystem for Linux"""
try:
with open('/proc/version') as fd:
linux_version = fd.read()
# hopefully no non-WSL Linux will ever mention 'Microsoft' in the kernel version:
return 'Microsoft' in linux_version
except: # noqa
# make sure to never ever crash due to this check.
return False
"write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
disk in the immediate future.
"""
def __init__(self, path, binary=False):
super().__init__(path, binary)
self.offset = 0
self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
self.last_sync = 0
self.pending_sync = None
if _is_WSL():
class SyncFile(BaseSyncFile):
# if we are on Microsoft's "Windows Subsytem for Linux", use the
# more generic BaseSyncFile to avoid issues like seen there:
# https://github.com/borgbackup/borg/issues/1961
pass
else:
# a real Linux, so we can do better. :)
class SyncFile(BaseSyncFile):
"""
Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability.
def write(self, data):
self.offset += self.fd.write(data)
offset = self.offset & ~PAGE_MASK
if offset >= self.last_sync + self.write_window:
"write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to
disk in the immediate future.
"""
def __init__(self, path, binary=False):
super().__init__(path, binary)
self.offset = 0
self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK
self.last_sync = 0
self.pending_sync = None
def write(self, data):
self.offset += self.fd.write(data)
offset = self.offset & ~PAGE_MASK
if offset >= self.last_sync + self.write_window:
self.fd.flush()
_sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
if self.pending_sync is not None:
_sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
self.pending_sync = self.last_sync
self.last_sync = offset
def sync(self):
self.fd.flush()
_sync_file_range(self.fileno, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE)
if self.pending_sync is not None:
_sync_file_range(self.fileno, self.pending_sync, self.last_sync - self.pending_sync,
SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER)
self.pending_sync = self.last_sync
self.last_sync = offset
def sync(self):
self.fd.flush()
os.fdatasync(self.fileno)
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
safe_fadvise(self.fileno, 0, 0, 'DONTNEED')
os.fdatasync(self.fileno)
# tell the OS that it does not need to cache what we just wrote,
# avoids spoiling the cache for the OS and other processes.
safe_fadvise(self.fileno, 0, 0, 'DONTNEED')