Initial work to build and run borg under windows

- Created a batch file to build borg on windows
- Adjusted setup.py to be runnable on windows and build the windows
extension
- Extracted the free space check to a function in the platform module
- Created the minimal needed (dummy) functions for the windows platform
module

.gitignore

@@ -14,9 +14,11 @@ src/borg/platform/darwin.c
 src/borg/platform/freebsd.c
 src/borg/platform/linux.c
 src/borg/platform/posix.c
+src/borg/platform/windows.c
 src/borg/_version.py
 *.egg-info
 *.pyc
+*.pyd
 *.so
 .idea/
 .cache/

MANIFEST.in

@@ -4,4 +4,4 @@
 exclude .coafile .editorconfig .gitattributes .gitignore .mailmap .travis.yml Vagrantfile
 prune .travis
 prune .github
-include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c
+include src/borg/platform/darwin.c src/borg/platform/freebsd.c src/borg/platform/linux.c src/borg/platform/posix.c src/borg/platform/windows.c

README_WINDOWS.rst

@@ -0,0 +1,19 @@
+Borg Native on Windows
+======================
+
+Build Requirements
+------------------
+
+- VC 14.0 Compiler
+- OpenSSL Library (https://slproweb.com/products/Win32OpenSSL.html)
+- Patience and a lot of coffee / beer
+
+What's working
+--------------
+
+- Borg does not crash if called with ``borg``
+- ``borg init --encryption none ./demoRepo`` runs without an error/warning.
+   Note that only relative paths work at the moment.
+
+
+

scripts/buildwin.bat

@@ -0,0 +1,20 @@
+
+REM Use the downloaded OpenSSL, for all other libraries the bundled version is used.
+REM On Appveyor different OpenSSL versions are available, therefore the directory contains the version information.
+set BORG_OPENSSL_PREFIX=C:\OpenSSL-v111-Win64
+set BORG_USE_BUNDLED_B2=YES
+set BORG_USE_BUNDLED_LZ4=YES
+set BORG_USE_BUNDLED_ZSTD=YES
+set BORG_USE_BUNDLED_XXHASH=YES
+
+REM Somehow on my machine rc.exe was not found. Adding the Windows Kit to the path worked.
+set PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64
+
+REM Run the build in the project directory.
+SET WORKPATH=%~dp0\..
+pushd %WORKPATH%
+
+python setup.py clean
+pip install -v -e .
+
+popd

setup.py

@@ -25,6 +25,8 @@
 import setup_crypto
 import setup_docs
 
+is_win32 = sys.platform.startswith('win32')
+
 # How the build process finds the system libs / uses the bundled code:
 #
 # 1. it will try to use (system) libs (see 1.1. and 1.2.),
@@ -60,6 +62,7 @@
 prefer_system_libxxhash = not bool(os.environ.get('BORG_USE_BUNDLED_XXHASH'))
 system_prefix_libxxhash = os.environ.get('BORG_LIBXXHASH_PREFIX')
 
+# Number of threads to use for cythonize, not used on windows
 cpu_threads = multiprocessing.cpu_count() if multiprocessing else 1
 
 # Are we building on ReadTheDocs?
@@ -97,6 +100,7 @@
 platform_linux_source = 'src/borg/platform/linux.pyx'
 platform_darwin_source = 'src/borg/platform/darwin.pyx'
 platform_freebsd_source = 'src/borg/platform/freebsd.pyx'
+platform_windows_source = 'src/borg/platform/windows.pyx'
 
 cython_sources = [
     compress_source,
@@ -110,6 +114,7 @@
     platform_linux_source,
     platform_freebsd_source,
     platform_darwin_source,
+    platform_windows_source,
 ]
 
 if cythonize:
@@ -199,9 +204,12 @@ def members_appended(*ds):
     linux_ext = Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])
     freebsd_ext = Extension('borg.platform.freebsd', [platform_freebsd_source])
     darwin_ext = Extension('borg.platform.darwin', [platform_darwin_source])
+    windows_ext = Extension('borg.platform.windows', [platform_windows_source])
 
-    if not sys.platform.startswith(('win32', )):
+    if not is_win32:
         ext_modules.append(posix_ext)
+    else:
+        ext_modules.append(windows_ext)
     if sys.platform == 'linux':
         ext_modules.append(linux_ext)
     elif sys.platform.startswith('freebsd'):
@@ -216,13 +224,15 @@ def members_appended(*ds):
 
     if cythonize and cythonizing:
         cython_opts = dict(
-            # compile .pyx extensions to .c in parallel
-            nthreads=cpu_threads + 1,
             # default language_level will be '3str' starting from Cython 3.0.0,
             # but old cython versions (< 0.29) do not know that, thus we use 3 for now.
             compiler_directives={'language_level': 3},
         )
-        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext], **cython_opts)
+        if not is_win32:
+            # compile .pyx extensions to .c in parallel, does not work on windows
+            cython_opts['nthreads'] = cpu_threads + 1
+
+        cythonize([posix_ext, linux_ext, freebsd_ext, darwin_ext, windows_ext], **cython_opts)
         ext_modules = cythonize(ext_modules, **cython_opts)
 
 

src/borg/platform/__init__.py

@@ -22,6 +22,10 @@
     from .posix import get_errno
     from .posix import uid2user, user2uid, gid2group, group2gid, getosusername
 
+else:
+    from .windows import process_alive, local_pid_alive
+    from .windows import uid2user, user2uid, gid2group, group2gid, getosusername
+
 if is_linux:  # pragma: linux only
     from .linux import API_VERSION as OS_API_VERSION
     from .linux import listxattr, getxattr, setxattr

src/borg/platform/base.py

@@ -4,6 +4,7 @@
 import uuid
 
 from borg.helpers import truncate_and_unlink
+from borg.platformflags import is_win32
 
 """
 platform base module
@@ -94,6 +95,10 @@ def get_flags(path, st, fd=None):
 
 
 def sync_dir(path):
+    if is_win32:
+        # Opening directories is not supported on windows.
+        # TODO: do we need to handle this in some other way?
+        return
     fd = os.open(path, os.O_RDONLY)
     try:
         os.fsync(fd)

src/borg/platform/windows.pyx

@@ -0,0 +1,60 @@
+import os
+import platform
+from functools import lru_cache
+
+
+cdef extern from 'windows.h':
+    ctypedef void* HANDLE
+    ctypedef int BOOL
+    ctypedef unsigned long DWORD
+
+    BOOL CloseHandle(HANDLE hObject)
+    HANDLE OpenProcess(DWORD dwDesiredAccess, BOOL bInheritHandle, DWORD dbProcessId)
+
+    cdef extern int PROCESS_QUERY_INFORMATION
+
+
+@lru_cache(maxsize=None)
+def uid2user(uid, default=None):
+    return default
+
+
+@lru_cache(maxsize=None)
+def user2uid(user, default=None):
+    return default
+
+
+@lru_cache(maxsize=None)
+def gid2group(gid, default=None):
+    return default
+    
+
+@lru_cache(maxsize=None)
+def group2gid(group, default=None):
+    return default
+    
+
+def getosusername():
+    """Return the os user name."""
+    return os.getlogin()
+
+
+def process_alive(host, pid, thread):
+    """
+    Check if the (host, pid, thread_id) combination corresponds to a potentially alive process.
+    """
+    if host.split('@')[0].lower() != platform.node().lower():
+        # Not running on the same node, assume running.
+        return True
+
+    # If the process can be opened, the process is alive.
+    handle = OpenProcess(PROCESS_QUERY_INFORMATION, False, pid)
+    if handle != NULL:
+        CloseHandle(handle)
+        return True
+    return False
+
+
+def local_pid_alive(pid):
+    """Return whether *pid* is alive."""
+    raise NotImplementedError

src/borg/repository.py

@@ -662,17 +662,12 @@ def check_free_space(self):
             else:
                 # Keep one full worst-case segment free in non-append-only mode
                 required_free_space += full_segment_size
+
         try:
-            st_vfs = os.statvfs(self.path)
+            free_space = shutil.disk_usage(self.path).free
         except OSError as os_error:
             logger.warning('Failed to check free space before committing: ' + str(os_error))
             return
-        except AttributeError:
-            # TODO move the call to statvfs to platform
-            logger.warning('Failed to check free space before committing: no statvfs method available')
-            return
-        # f_bavail: even as root - don't touch the Federal Block Reserve!
-        free_space = st_vfs.f_bavail * st_vfs.f_frsize
         logger.debug('check_free_space: required bytes {}, free bytes {}'.format(required_free_space, free_space))
         if free_space < required_free_space:
             if self.created: