Merge pull request #4733 from jrast/win10-cibuild

Windows 10 CI-Build

.appveyor.yml

@@ -0,0 +1,27 @@
+version: '{build}'
+
+environment:  
+  matrix:
+    - PYTHON: C:\Python37-x64
+
+# Disable automatic builds
+build: off
+
+# Build artifacts: all wheel and exe files in the dist folder
+artifacts:
+  - path: 'dist\*.whl'
+  - path: 'dist\*.exe'
+
+install:
+- ps: scripts\win-download-openssl.ps1
+- ps: |
+    & $env:PYTHON\python.exe -m venv borg-env
+    borg-env\Scripts\activate.ps1
+    python -m pip install -U pip
+    pip install -r requirements.d/development.txt
+    pip install wheel pyinstaller
+
+build_script:
+- ps: |
+    borg-env\Scripts\activate.ps1
+    scripts\win-build.ps1

.gitignore

@@ -2,6 +2,7 @@ MANIFEST
 docs/_build
 build
 dist
+external
 borg-env
 .tox
 src/borg/compress.c

README_WINDOWS.rst

@@ -8,7 +8,9 @@ Build Requirements
 ------------------
 
 - VC 14.0 Compiler
-- OpenSSL Library v1.1.1c, 64bit (available at https://slproweb.com/products/Win32OpenSSL.html)
+- OpenSSL Library v1.1.1c, 64bit (available at https://github.com/python/cpython-bin-deps)
+  Please use the `win-download-openssl.ps1` script to download and extract the library to
+  the correct location. See also the OpenSSL section below.
 - Patience and a lot of coffee / beer
 
 What's working
@@ -32,3 +34,13 @@ What's NOT working
 
 - Extracting a backup which was created on windows machine on a non windows machine will fail.
 - And many things more.
+
+
+OpenSSL, Windows and Python
+---------------------------
+Windows does not ship OpenSSL by default, so we need to get the library from somewhere else.
+However, a default python installation does include `libcrypto` which is required by borg.
+The only things which are missing to build borg are the header and `*.lib` files.
+Luckily the python developers provide all required files in a separate repository.
+The `win-download-openssl.ps1` script can be used to download the package from
+https://github.com/python/cpython-bin-deps and extract the files to the correct location.

scripts/borg.exe.spec

@@ -3,17 +3,27 @@
 
 import os, sys
 
-basepath = '/vagrant/borg/borg'
+is_win32 = sys.platform.startswith('win32')
+
+# Note: SPEC contains the spec file argument given to pyinstaller
+here = os.path.dirname(os.path.abspath(SPEC))
+
+if is_win32:
+    basepath = os.path.abspath(os.path.join(here, '..'))
+    hiddenimports = []
+else:
+    basepath = '/vagrant/borg/borg'
+    hiddenimports = ['borg.platform.posix']
 
 block_cipher = None
 
-a = Analysis([os.path.join(basepath, 'src/borg/__main__.py'), ],
+a = Analysis([os.path.join(basepath, 'src', 'borg', '__main__.py'), ],
              pathex=[basepath, ],
              binaries=[],
              datas=[
-                 ('../src/borg/paperkey.html', 'borg'),
+                (os.path.join(basepath, 'src', 'borg', 'paperkey.html'), 'borg'),
              ],
-             hiddenimports=['borg.platform.posix'],
+             hiddenimports=hiddenimports,
              hookspath=[],
              runtime_hooks=[],
              excludes=[
@@ -39,7 +49,7 @@ exe = EXE(pyz,
           debug=False,
           strip=False,
           upx=True,
-          console=True )
+          console=True)
 
 if False:
     # Enable this block to build a directory-based binary instead of

scripts/buildwin.bat

@@ -1,20 +0,0 @@
-
-REM Use the downloaded OpenSSL, for all other libraries the bundled version is used.
-REM On Appveyor different OpenSSL versions are available, therefore the directory contains the version information.
-set BORG_OPENSSL_PREFIX=C:\OpenSSL-v111-Win64
-set BORG_USE_BUNDLED_B2=YES
-set BORG_USE_BUNDLED_LZ4=YES
-set BORG_USE_BUNDLED_ZSTD=YES
-set BORG_USE_BUNDLED_XXHASH=YES
-
-REM Somehow on my machine rc.exe was not found. Adding the Windows Kit to the path worked.
-set PATH=%PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.18362.0\x64
-
-REM Run the build in the project directory.
-SET WORKPATH=%~dp0\..
-pushd %WORKPATH%
-
-python setup.py clean
-pip install -v -e .
-
-popd

scripts/win-build.ps1

@@ -0,0 +1,18 @@
+# Build a wheel and single file executable
+
+# Configure the build environment
+& $PSScriptRoot\win-setup-build-env.ps1
+
+# Clean the old build
+python setup.py clean
+
+# Build the extension inplace
+python setup.py build_ext --inplace
+
+# Run pip install to install install_requires of borg.
+pip install -v -e .
+
+# Build the wheel
+python setup.py bdist_wheel
+
+pyinstaller -y scripts/borg.exe.spec

scripts/win-download-openssl.ps1

@@ -0,0 +1,15 @@
+# Download and extract the prebuilt openssl libraries provided by the python developers.
+# The file is extracted to the .\external directory.
+
+$url = "https://github.com/python/cpython-bin-deps/archive/openssl-bin-1.1.1c.zip"
+$dest = "external"
+
+$ErrorActionPreference = "Stop"
+
+Write-Output "Downloading OpenSSL from cpython-bin-deps repository ..." 
+Invoke-WebRequest $url -OutFile openssl.zip
+
+Write-Output "Extracting OpenSSL"
+Expand-Archive -Path openssl.zip -DestinationPath $dest -Force
+
+Remove-Item -Path openssl.zip

scripts/win-setup-build-env.ps1

@@ -0,0 +1,18 @@
+# Configure the environment such that borg can be built and run.
+# Note that building borg requires OpenSSL which is not available by default.
+# Use the win-download-openssl.ps1 script to get correct OpenSSL version.
+
+$opensslPath = Resolve-Path "$PSScriptRoot\..\external\cpython-bin-deps-openssl-bin-1.1.1c\$env:PROCESSOR_ARCHITECTURE"
+if(!(Test-Path $opensslPath)) {
+    Write-Host "OpenSSL not found! Please run win-download-openssl.ps1 and check if your platform is supported."
+    exit
+}
+
+$env:BORG_OPENSSL_PREFIX = $opensslPath
+$env:BORG_USE_BUNDLED_B2 = "YES"
+$env:BORG_USE_BUNDLED_LZ4 = "YES"
+$env:BORG_USE_BUNDLED_ZSTD = "YES"
+$env:BORG_USE_BUNDLED_XXHASH = "YES"
+
+Write-Host "Environment configured for borg. The following variables where set:"
+Write-Host ( Get-ChildItem Env: | Where-Object { $_.Name.StartsWith("BORG_") } | Out-String )

setup_crypto.py

@@ -1,6 +1,9 @@
 # Support code for building a C extension with crypto code
 
 import os
+import sys
+
+is_win32 = sys.platform.startswith('win32')
 
 
 def multi_join(paths, *path_segments):
@@ -11,9 +14,16 @@ def multi_join(paths, *path_segments):
 def crypto_ext_kwargs(pc, system_prefix):
     if system_prefix:
         print('Detected OpenSSL [via BORG_OPENSSL_PREFIX]')
+        if is_win32:
+            lib_dir = system_prefix
+            lib_name = 'libcrypto'
+        else:
+            lib_dir = os.path.join(system_prefix, 'lib')
+            lib_name = 'crypto'
+
         return dict(include_dirs=[os.path.join(system_prefix, 'include')],
-                    library_dirs=[os.path.join(system_prefix, 'lib')],
-                    libraries=['crypto'])
+                    library_dirs=[lib_dir],
+                    libraries=[lib_name])
 
     if pc and pc.exists('libcrypto'):
         print('Detected OpenSSL [via pkg-config]')

src/borg/__main__.py

@@ -1,2 +1,15 @@
+import sys
+import os
+
+# On windows loading the bundled libcrypto dll fails if the folder
+# containing the dll is not in the search path. The dll is shipped
+# with python in the "DLLs" folder, so let's add this folder
+# to the path. The folder is always in sys.path, get it from there.
+if sys.platform.startswith('win32'):
+    # Keep it an iterable to support multiple folder which contain "DLLs".
+    dll_path = (p for p in sys.path if 'DLLs' in os.path.normpath(p).split(os.path.sep))
+    os.environ['PATH'] = os.pathsep.join(dll_path) + os.pathsep + os.environ['PATH']
+
+
 from borg.archiver import main
 main()