diff --git a/Vagrantfile b/Vagrantfile index fdb2feae1..669c0975a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,13 +7,6 @@ $cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus $xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers $wmem = $xdistn * 256 # give the VM additional memory for workers [MB] -def packages_prepare_wheezy - return <<-EOF - # debian 7 wheezy does not have lz4, but it is available from wheezy-backports: - echo "deb http://http.debian.net/debian wheezy-backports main" > /etc/apt/sources.list.d/wheezy-backports.list - EOF -end - def packages_debianoid return <<-EOF if id "vagrant" >/dev/null 2>&1; then @@ -35,202 +28,16 @@ def packages_debianoid apt-get install -y python3-dev python3-setuptools # for building python: apt-get install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev - # this way it works on older dists (like ubuntu 12.04) also: - # for python 3.2 on ubuntu 12.04 we need pip<8 and virtualenv<14 as - # newer versions are not compatible with py 3.2 any more. - easy_install3 'pip<8.0' - pip3 install 'virtualenv<14.0' + easy_install3 'pip' + pip3 install 'virtualenv' touch $home_dir/.bash_profile ; chown $username $home_dir/.bash_profile EOF end -def packages_redhatted - return <<-EOF - yum install -y epel-release - yum update -y - # for building borgbackup and dependencies: - yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig - usermod -a -G fuse vagrant - chgrp fuse /dev/fuse - chmod 666 /dev/fuse - yum install -y fakeroot gcc git patch - # needed to compile msgpack-python (otherwise it will use slow fallback code): - yum install -y gcc-c++ - # for building python: - yum install -y zlib-devel bzip2-devel ncurses-devel readline-devel xz xz-devel sqlite-devel - #yum install -y python-pip - #pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_darwin - return <<-EOF - # install all the (security and other) updates - sudo softwareupdate --ignore iTunesX - sudo softwareupdate --ignore iTunes - sudo softwareupdate --install --all - # get osxfuse 3.x release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.6.3/osxfuse-3.6.3.dmg >osxfuse.dmg - MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.6.3.pkg" -target / - sudo chown -R vagrant /usr/local # brew must be able to create stuff here - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - brew update - brew install openssl - brew install lz4 - brew install xz # required for python lzma module - brew install fakeroot - brew install git - brew install pkg-config - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_freebsd - return <<-EOF - # VM has no hostname set - hostname freebsd - # install all the (security and other) updates, base system - freebsd-update --not-running-from-cron fetch install - # for building borgbackup and dependencies: - pkg install -y openssl liblz4 fusefs-libs pkgconf - pkg install -y git bash - # for building python: - pkg install -y sqlite3 - # make bash default / work: - chsh -s bash vagrant - mount -t fdescfs fdesc /dev/fd - echo 'fdesc /dev/fd fdescfs rw 0 0' >> /etc/fstab - # make FUSE work - echo 'fuse_load="YES"' >> /boot/loader.conf - echo 'vfs.usermount=1' >> /etc/sysctl.conf - kldload fuse - sysctl vfs.usermount=1 - pw groupmod operator -M vagrant - # /dev/fuse has group operator - chmod 666 /dev/fuse - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - # install all the (security and other) updates, packages - pkg update - yes | pkg upgrade - EOF -end - -def packages_openbsd - return <<-EOF - echo 'installpath = http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' > /etc/pkg.conf - echo 'export PKG_PATH=http://ftp.hostserver.de/pub/OpenBSD/6.0/packages/amd64/' >> ~/.profile - . ~/.profile - pkg_add bash - chsh -s /usr/local/bin/bash vagrant - pkg_add openssl - pkg_add lz4 - pkg_add git # no fakeroot - pkg_add py3-setuptools - # TODO: python 3.5 or 3.6 - ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3 - ln -sf /usr/local/bin/python3.4 /usr/local/bin/python - easy_install-3.4 pip - pip3 install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_netbsd - return <<-EOF - hostname netbsd # the box we use has an invalid hostname - PKG_PATH="ftp://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/amd64/6.1.5/All/" - export PKG_PATH - pkg_add mozilla-rootcerts lz4 git bash - chsh -s bash vagrant - mkdir -p /usr/local/opt/lz4/include - mkdir -p /usr/local/opt/lz4/lib - ln -s /usr/pkg/include/lz4*.h /usr/local/opt/lz4/include/ - ln -s /usr/pkg/lib/liblz4* /usr/local/opt/lz4/lib/ - touch /etc/openssl/openssl.cnf # avoids a flood of "can't open ..." - mozilla-rootcerts install - pkg_add pkg-config # avoids some "pkg-config missing" error msg, even without fuse pkg - # pkg_add fuse # llfuse supports netbsd, but is still buggy. - # https://bitbucket.org/nikratio/python-llfuse/issues/70/perfuse_open-setsockopt-no-buffer-space - # TODO: python 3.5 or 3.6 - pkg_add python34 py34-setuptools - ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python - ln -s /usr/pkg/bin/python3.4 /usr/pkg/bin/python3 - easy_install-3.4 pip - pip install virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -def packages_openindiana - return <<-EOF - #pkg update # XXX needs separate provisioning step + reboot - # TODO python 3.5 or 3.6 - pkg install python-34 clang-3.4 lz4 git - python3 -m ensurepip - pip3 install -U setuptools pip wheel virtualenv - touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile - EOF -end - -# Install required cygwin packages and configure environment -# -# Microsoft/EdgeOnWindows10 image has MLS-OpenSSH installed by default, -# which is based on cygwin x86_64 but should not be used together with cygwin. -# In order to have have cygwin compatible bash 'ImagePath' is replaced with -# cygrunsrv of newly installed cygwin -# -# supported cygwin versions: -# x86_64 -# x86 -def packages_cygwin(version) - setup_exe = "setup-#{version}.exe" - - return <<-EOF - mkdir -p /cygdrive/c/cygwin - powershell -Command '$client = new-object System.Net.WebClient; $client.DownloadFile("https://www.cygwin.com/#{setup_exe}","C:\\cygwin\\#{setup_exe}")' - echo ' - REM --- Change to use different CygWin platform and final install path - set CYGSETUP=#{setup_exe} - REM --- Install build version of CygWin in a subfolder - set OURPATH=%cd% - set CYGBUILD="C:\\cygwin\\CygWin" - set CYGMIRROR=http://mirrors.kernel.org/sourceware/cygwin/ - set BASEPKGS=openssh,rsync - set BUILDPKGS=python3,python3-setuptools,python-devel,binutils,gcc-g++,libopenssl,openssl-devel,git,make,liblz4-devel,liblz4_1,curl - %CYGSETUP% -q -B -o -n -R %CYGBUILD% -L -D -s %CYGMIRROR% -P %BASEPKGS%,%BUILDPKGS% - cd /d C:\\cygwin\\CygWin\\bin - regtool set /HKLM/SYSTEM/CurrentControlSet/Services/OpenSSHd/ImagePath "C:\\cygwin\\CygWin\\bin\\cygrunsrv.exe" - bash -c "ssh-host-config --no" - bash -c "chown sshd_server /cygdrive/c/cygwin/CygWin/var/empty" - ' > /cygdrive/c/cygwin/install.bat - - echo "alias mkdir='mkdir -p'" > ~/.profile - echo "export CYGWIN_ROOT=/cygdrive/c/cygwin/CygWin" >> ~/.profile - echo 'export PATH=$CYGWIN_ROOT/bin:$PATH' >> ~/.profile - - echo '' > ~/.bash_profile - - cmd.exe /c 'setx /m PATH "C:\\cygwin\\CygWin\\bin;%PATH%"' - source ~/.profile - cd /cygdrive/c/cygwin && cmd.exe /c install.bat - - echo 'db_home: windows' > $CYGWIN_ROOT/etc/nsswitch.conf - EOF -end - -def install_cygwin_venv - return <<-EOF - python3 -m ensurepip -U --default-pip - pip install virtualenv - EOF -end - def install_pyenv(boxname) script = <<-EOF curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash - echo 'export PATH="$HOME/.pyenv/bin:/vagrant/borg:$PATH"' >> ~/.bash_profile + echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> ~/.bash_profile echo 'eval "$(pyenv init -)"' >> ~/.bash_profile echo 'eval "$(pyenv virtualenv-init -)"' >> ~/.bash_profile echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile @@ -322,6 +129,7 @@ def build_binary_with_pyinstaller(boxname) . borg-env/bin/activate cd borg pyinstaller --clean --distpath=/vagrant/borg scripts/borg.exe.spec + echo 'export PATH="/vagrant/borg:$PATH"' >> ~/.bash_profile EOF end @@ -371,46 +179,6 @@ Vagrant.configure(2) do |config| v.cpus = $cpus end - # Linux - config.vm.define "centos7_64" do |b| - b.vm.box = "centos/7" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos7_64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos7_64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos7_64") - end - - config.vm.define "centos6_32" do |b| - b.vm.box = "centos6-32" - b.vm.provider :virtualbox do |v| - v.memory = 768 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_32") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_32") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_32") - end - - config.vm.define "centos6_64" do |b| - b.vm.box = "centos6-64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("centos6_64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("centos6_64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("centos6_64") - end - config.vm.define "xenial64" do |b| b.vm.box = "ubuntu/xenial64" b.vm.provider :virtualbox do |v| @@ -422,17 +190,6 @@ Vagrant.configure(2) do |config| b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("xenial64") end - config.vm.define "trusty64" do |b| - b.vm.box = "ubuntu/trusty64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("trusty64") - end - config.vm.define "stretch64" do |b| b.vm.box = "debian/stretch64" b.vm.provider :virtualbox do |v| @@ -441,155 +198,11 @@ Vagrant.configure(2) do |config| b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("stretch64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) + b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() + b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("stretch64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("stretch64") end - config.vm.define "jessie64" do |b| - b.vm.box = "debian/jessie64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("jessie64") - end - - config.vm.define "wheezy32" do |b| - b.vm.box = "boxcutter/debian7-i386" - b.vm.provider :virtualbox do |v| - v.memory = 768 + $wmem - end - b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy32") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy32") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy32") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy32") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy32") - end - - config.vm.define "wheezy64" do |b| - b.vm.box = "boxcutter/debian7" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy - b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("wheezy64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("wheezy64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("wheezy64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("wheezy64") - end - - # OS X - config.vm.define "darwin64" do |b| - b.vm.box = "jhcook/yosemite-clitools" - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64'] - v.customize ['modifyvm', :id, '--paravirtprovider', 'default'] - # Adjust CPU settings according to - # https://github.com/geerlingguy/macos-virtualbox-vm - v.customize ['modifyvm', :id, '--cpuidset', - '00000001', '000306a9', '00020800', '80000201', '178bfbff'] - # Disable USB variant requiring Virtualbox proprietary extension pack - v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] - end - b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") - b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("darwin64") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("darwin64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("darwin64") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("darwin64") - end - - # BSD - # note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start. - config.vm.define "freebsd64" do |b| - b.vm.box = "freebsd/FreeBSD-10.3-RELEASE" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.ssh.shell = "sh" - b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd - b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("freebsd") - b.vm.provision "install pythons", :type => :shell, :privileged => false, :inline => install_pythons("freebsd") - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_pyenv_venv("freebsd") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(true) - b.vm.provision "install pyinstaller", :type => :shell, :privileged => false, :inline => install_pyinstaller() - b.vm.provision "build binary with pyinstaller", :type => :shell, :privileged => false, :inline => build_binary_with_pyinstaller("freebsd") - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("freebsd") - end - - config.vm.define "openbsd64" do |b| - b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.ssh.shell = "sh" - b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") - end - - config.vm.define "netbsd64" do |b| - b.vm.box = "netbsd70-64" - b.vm.provider :virtualbox do |v| - v.memory = 1024 + $wmem - end - b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("netbsd64") - end - - # rsync on openindiana has troubles, does not set correct owner for /vagrant/borg and thus gives lots of - # permission errors. can be manually fixed in the VM by: sudo chown -R vagrant /vagrant/borg ; then rsync again. - config.vm.define "openindiana64" do |b| - b.vm.box = "openindiana/hipster" - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - end - b.vm.provision "packages openindiana", :type => :shell, :inline => packages_openindiana - b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openindiana64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openindiana64") - end - - config.vm.define "windows10" do |b| - b.vm.box = "Microsoft/EdgeOnWindows10" - b.vm.guest = :windows - b.vm.boot_timeout = 180 - b.vm.graceful_halt_timeout = 120 - - b.ssh.shell = "sh -l" - b.ssh.username = "IEUser" - b.ssh.password = "Passw0rd!" - b.ssh.insert_key = false - - b.vm.provider :virtualbox do |v| - v.memory = 1536 + $wmem - #v.gui = true - end - - # fix permissions placeholder - b.vm.provision "fix perms", :type => :shell, :privileged => false, :inline => "echo 'fix permission placeholder'" - - b.vm.provision "packages cygwin", :type => :shell, :privileged => false, :inline => packages_cygwin("x86_64") - b.vm.provision :reload - b.vm.provision "cygwin install pip", :type => :shell, :privileged => false, :inline => install_cygwin_venv - b.vm.provision "cygwin build env", :type => :shell, :privileged => false, :inline => build_sys_venv("windows10") - b.vm.provision "cygwin install borg", :type => :shell, :privileged => false, :inline => install_borg(false) - b.vm.provision "cygwin run tests", :type => :shell, :privileged => false, :inline => run_tests("windows10") - end + # TODO: create more VMs with python 3.5+ and openssl 1.1. + # See branch 1.1-maint for a better equipped Vagrantfile (but still on py34 and openssl 1.0). end