import os import re import stat from .posix import posix_acl_use_stored_uid_gid from .posix import user2uid, group2gid from ..helpers import workarounds from ..helpers import safe_decode, safe_encode from .base import SyncFile as BaseSyncFile from .base import safe_fadvise from .xattr import _listxattr_inner, _getxattr_inner, _setxattr_inner, split_string0 try: from .syncfilerange import sync_file_range, SYNC_FILE_RANGE_WRITE, SYNC_FILE_RANGE_WAIT_BEFORE, SYNC_FILE_RANGE_WAIT_AFTER SYNC_FILE_RANGE_LOADED = True except ImportError: SYNC_FILE_RANGE_LOADED = False from libc cimport errno API_VERSION = '1.2_05' cdef extern from "sys/xattr.h": ssize_t c_listxattr "listxattr" (const char *path, char *list, size_t size) ssize_t c_llistxattr "llistxattr" (const char *path, char *list, size_t size) ssize_t c_flistxattr "flistxattr" (int filedes, char *list, size_t size) ssize_t c_getxattr "getxattr" (const char *path, const char *name, void *value, size_t size) ssize_t c_lgetxattr "lgetxattr" (const char *path, const char *name, void *value, size_t size) ssize_t c_fgetxattr "fgetxattr" (int filedes, const char *name, void *value, size_t size) int c_setxattr "setxattr" (const char *path, const char *name, const void *value, size_t size, int flags) int c_lsetxattr "lsetxattr" (const char *path, const char *name, const void *value, size_t size, int flags) int c_fsetxattr "fsetxattr" (int filedes, const char *name, const void *value, size_t size, int flags) cdef extern from "sys/types.h": int ACL_TYPE_ACCESS int ACL_TYPE_DEFAULT cdef extern from "sys/acl.h": ctypedef struct _acl_t: pass ctypedef _acl_t *acl_t int acl_free(void *obj) acl_t acl_get_file(const char *path, int type) acl_t acl_get_fd(int fd) int acl_set_file(const char *path, int type, acl_t acl) int acl_set_fd(int fd, acl_t acl) acl_t acl_from_text(const char *buf) char *acl_to_text(acl_t acl, ssize_t *len) cdef extern from "acl/libacl.h": int acl_extended_file_nofollow(const char *path) int acl_extended_fd(int fd) cdef extern from "linux/fs.h": # ioctls int FS_IOC_SETFLAGS int FS_IOC_GETFLAGS # inode flags int FS_NODUMP_FL int FS_IMMUTABLE_FL int FS_APPEND_FL int FS_COMPR_FL cdef extern from "sys/ioctl.h": int ioctl(int fildes, int request, ...) cdef extern from "unistd.h": int _SC_PAGESIZE long sysconf(int name) cdef extern from "string.h": char *strerror(int errnum) _comment_re = re.compile(' *#.*', re.M) def listxattr(path, *, follow_symlinks=False): def func(path, buf, size): if isinstance(path, int): return c_flistxattr(path, buf, size) else: if follow_symlinks: return c_listxattr(path, buf, size) else: return c_llistxattr(path, buf, size) n, buf = _listxattr_inner(func, path) return [name for name in split_string0(buf[:n]) if name and not name.startswith(b'system.posix_acl_')] def getxattr(path, name, *, follow_symlinks=False): def func(path, name, buf, size): if isinstance(path, int): return c_fgetxattr(path, name, buf, size) else: if follow_symlinks: return c_getxattr(path, name, buf, size) else: return c_lgetxattr(path, name, buf, size) n, buf = _getxattr_inner(func, path, name) return bytes(buf[:n]) def setxattr(path, name, value, *, follow_symlinks=False): def func(path, name, value, size): flags = 0 if isinstance(path, int): return c_fsetxattr(path, name, value, size, flags) else: if follow_symlinks: return c_setxattr(path, name, value, size, flags) else: return c_lsetxattr(path, name, value, size, flags) _setxattr_inner(func, path, name, value) BSD_TO_LINUX_FLAGS = { stat.UF_NODUMP: FS_NODUMP_FL, stat.UF_IMMUTABLE: FS_IMMUTABLE_FL, stat.UF_APPEND: FS_APPEND_FL, stat.UF_COMPRESSED: FS_COMPR_FL, } def set_flags(path, bsd_flags, fd=None): if fd is None: st = os.stat(path, follow_symlinks=False) if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): # see comment in get_flags() return cdef int flags = 0 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): if bsd_flags & bsd_flag: flags |= linux_flag open_fd = fd is None if open_fd: fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) try: if ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1: error_number = errno.errno if error_number != errno.EOPNOTSUPP: raise OSError(error_number, strerror(error_number).decode(), path) finally: if open_fd: os.close(fd) def get_flags(path, st, fd=None): if stat.S_ISBLK(st.st_mode) or stat.S_ISCHR(st.st_mode) or stat.S_ISLNK(st.st_mode): # avoid opening devices files - trying to open non-present devices can be rather slow. # avoid opening symlinks, O_NOFOLLOW would make the open() fail anyway. return 0 cdef int linux_flags open_fd = fd is None if open_fd: try: fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) except OSError: return 0 try: if ioctl(fd, FS_IOC_GETFLAGS, &linux_flags) == -1: return 0 finally: if open_fd: os.close(fd) bsd_flags = 0 for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): if linux_flags & linux_flag: bsd_flags |= bsd_flag return bsd_flags def acl_use_local_uid_gid(acl): """Replace the user/group field with the local uid/gid if possible """ assert isinstance(acl, bytes) entries = [] for entry in safe_decode(acl).split('\n'): if entry: fields = entry.split(':') if fields[0] == 'user' and fields[1]: fields[1] = str(user2uid(fields[1], fields[3])) elif fields[0] == 'group' and fields[1]: fields[1] = str(group2gid(fields[1], fields[3])) entries.append(':'.join(fields[:3])) return safe_encode('\n'.join(entries)) cdef acl_append_numeric_ids(acl): """Extend the "POSIX 1003.1e draft standard 17" format with an additional uid/gid field """ assert isinstance(acl, bytes) entries = [] for entry in _comment_re.sub('', safe_decode(acl)).split('\n'): if entry: type, name, permission = entry.split(':') if name and type == 'user': entries.append(':'.join([type, name, permission, str(user2uid(name, name))])) elif name and type == 'group': entries.append(':'.join([type, name, permission, str(group2gid(name, name))])) else: entries.append(entry) return safe_encode('\n'.join(entries)) cdef acl_numeric_ids(acl): """Replace the "POSIX 1003.1e draft standard 17" user/group field with uid/gid """ assert isinstance(acl, bytes) entries = [] for entry in _comment_re.sub('', safe_decode(acl)).split('\n'): if entry: type, name, permission = entry.split(':') if name and type == 'user': uid = str(user2uid(name, name)) entries.append(':'.join([type, uid, permission, uid])) elif name and type == 'group': gid = str(group2gid(name, name)) entries.append(':'.join([type, gid, permission, gid])) else: entries.append(entry) return safe_encode('\n'.join(entries)) def acl_get(path, item, st, numeric_ids=False, fd=None): cdef acl_t default_acl = NULL cdef acl_t access_acl = NULL cdef char *default_text = NULL cdef char *access_text = NULL cdef int ret = 0 if isinstance(path, str): path = os.fsencode(path) if fd is not None: ret = acl_extended_fd(fd) else: ret = acl_extended_file_nofollow(path) if ret < 0: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) if ret == 0: # there is no ACL defining permissions other than those defined by the traditional file permission bits. # note: this should also be the case for symlink fs objects, as they can not have ACLs. return if numeric_ids: converter = acl_numeric_ids else: converter = acl_append_numeric_ids try: if fd is not None: access_acl = acl_get_fd(fd) else: access_acl = acl_get_file(path, ACL_TYPE_ACCESS) if access_acl == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) access_text = acl_to_text(access_acl, NULL) if access_text == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) item['acl_access'] = converter(access_text) finally: acl_free(access_text) acl_free(access_acl) if stat.S_ISDIR(st.st_mode): # only directories can have a default ACL. there is no fd-based api to get it. try: default_acl = acl_get_file(path, ACL_TYPE_DEFAULT) if default_acl == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) default_text = acl_to_text(default_acl, NULL) if default_text == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) item['acl_default'] = converter(default_text) finally: acl_free(default_text) acl_free(default_acl) def acl_set(path, item, numeric_ids=False, fd=None): cdef acl_t access_acl = NULL cdef acl_t default_acl = NULL if stat.S_ISLNK(item.get('mode', 0)): # Linux does not support setting ACLs on symlinks return if isinstance(path, str): path = os.fsencode(path) if numeric_ids: converter = posix_acl_use_stored_uid_gid else: converter = acl_use_local_uid_gid access_text = item.get('acl_access') if access_text is not None: try: access_acl = acl_from_text(converter(access_text)) if access_acl == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) if fd is not None: if acl_set_fd(fd, access_acl) == -1: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) else: if acl_set_file(path, ACL_TYPE_ACCESS, access_acl) == -1: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) finally: acl_free(access_acl) default_text = item.get('acl_default') if default_text is not None: try: default_acl = acl_from_text(converter(default_text)) if default_acl == NULL: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) # only directories can get a default ACL. there is no fd-based api to set it. if acl_set_file(path, ACL_TYPE_DEFAULT, default_acl) == -1: raise OSError(errno.errno, os.strerror(errno.errno), os.fsdecode(path)) finally: acl_free(default_acl) cdef _sync_file_range(fd, offset, length, flags): assert offset & PAGE_MASK == 0, "offset %d not page-aligned" % offset assert length & PAGE_MASK == 0, "length %d not page-aligned" % length if sync_file_range(fd, offset, length, flags) != 0: raise OSError(errno.errno, os.strerror(errno.errno)) safe_fadvise(fd, offset, length, 'DONTNEED') cdef unsigned PAGE_MASK = sysconf(_SC_PAGESIZE) - 1 if 'basesyncfile' in workarounds or not SYNC_FILE_RANGE_LOADED: class SyncFile(BaseSyncFile): # if we are on platforms with a broken or not implemented sync_file_range, # use the more generic BaseSyncFile to avoid issues. # see basesyncfile description in our docs for details. pass else: # a real Linux, so we can do better. :) class SyncFile(BaseSyncFile): """ Implemented using sync_file_range for asynchronous write-out and fdatasync for actual durability. "write-out" means that dirty pages (= data that was written) are submitted to an I/O queue and will be send to disk in the immediate future. """ def __init__(self, path, *, fd=None, binary=False): super().__init__(path, fd=fd, binary=binary) self.offset = 0 self.write_window = (16 * 1024 ** 2) & ~PAGE_MASK self.last_sync = 0 self.pending_sync = None def write(self, data): self.offset += self.f.write(data) offset = self.offset & ~PAGE_MASK if offset >= self.last_sync + self.write_window: self.f.flush() _sync_file_range(self.fd, self.last_sync, offset - self.last_sync, SYNC_FILE_RANGE_WRITE) if self.pending_sync is not None: _sync_file_range(self.fd, self.pending_sync, self.last_sync - self.pending_sync, SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WAIT_AFTER) self.pending_sync = self.last_sync self.last_sync = offset def sync(self): self.f.flush() os.fdatasync(self.fd) # tell the OS that it does not need to cache what we just wrote, # avoids spoiling the cache for the OS and other processes. safe_fadvise(self.fd, 0, 0, 'DONTNEED')