diff --git a/Vagrantfile b/Vagrantfile index e42f73330..e71861098 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -3,6 +3,10 @@ # Automated creation of testing environments / binaries on misc. platforms +$cpus = Integer(ENV.fetch('VMCPUS', '4')) # create VMs with that many cpus +$xdistn = Integer(ENV.fetch('XDISTN', '4')) # dispatch tests to that many pytest workers +$wmem = $xdistn * 256 # give the VM additional memory for workers [MB] + def packages_prepare_wheezy return <<-EOF # debian 7 wheezy does not have lz4, but it is available from wheezy-backports: @@ -209,7 +213,7 @@ def install_cygwin_venv end def install_pyenv(boxname) - return <<-EOF + script = <<-EOF curl -s -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash echo 'export PATH="$HOME/.pyenv/bin:/vagrant/borg:$PATH"' >> ~/.bash_profile echo 'eval "$(pyenv init -)"' >> ~/.bash_profile @@ -217,6 +221,8 @@ def install_pyenv(boxname) echo 'export PYTHON_CONFIGURE_OPTS="--enable-shared"' >> ~/.bash_profile echo 'export LANG=en_US.UTF-8' >> ~/.bash_profile EOF + script += "echo 'export XDISTN=%d' >> ~/.bash_profile\n" % [$xdistn] + return script end def fix_pyenv_darwin(boxname) @@ -263,10 +269,10 @@ def install_borg(fuse) pip install -U wheel # upgrade wheel, too old for 3.5 cd borg # clean up (wrong/outdated) stuff we likely got via rsync: - rm -f borg/*.so borg/*.cpy* - rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c - rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__ + rm -rf __pycache__ + find src -name '__pycache__' -exec rm -rf {} \; pip install -r requirements.d/development.txt + python setup.py clean EOF if fuse script += <<-EOF @@ -348,14 +354,14 @@ Vagrant.configure(2) do |config| config.vm.provider :virtualbox do |v| #v.gui = true - v.cpus = 1 + v.cpus = $cpus end # Linux config.vm.define "centos7_64" do |b| b.vm.box = "centos/7" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos7_64") @@ -368,7 +374,7 @@ Vagrant.configure(2) do |config| config.vm.define "centos6_32" do |b| b.vm.box = "centos6-32" b.vm.provider :virtualbox do |v| - v.memory = 1024 + v.memory = 768 + $wmem end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_32") @@ -381,7 +387,7 @@ Vagrant.configure(2) do |config| config.vm.define "centos6_64" do |b| b.vm.box = "centos6-64" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "install system packages", :type => :shell, :inline => packages_redhatted b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("centos6_64") @@ -394,7 +400,7 @@ Vagrant.configure(2) do |config| config.vm.define "xenial64" do |b| b.vm.box = "ubuntu/xenial64" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("xenial64") @@ -405,7 +411,7 @@ Vagrant.configure(2) do |config| config.vm.define "trusty64" do |b| b.vm.box = "ubuntu/trusty64" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("trusty64") @@ -416,7 +422,7 @@ Vagrant.configure(2) do |config| config.vm.define "jessie64" do |b| b.vm.box = "debian/jessie64" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("jessie64") @@ -427,7 +433,7 @@ Vagrant.configure(2) do |config| config.vm.define "wheezy32" do |b| b.vm.box = "boxcutter/debian7-i386" b.vm.provider :virtualbox do |v| - v.memory = 1024 + v.memory = 768 + $wmem end b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid @@ -442,6 +448,9 @@ Vagrant.configure(2) do |config| config.vm.define "wheezy64" do |b| b.vm.box = "boxcutter/debian7" + b.vm.provider :virtualbox do |v| + v.memory = 1024 + $wmem + end b.vm.provision "packages prepare wheezy", :type => :shell, :inline => packages_prepare_wheezy b.vm.provision "packages debianoid", :type => :shell, :inline => packages_debianoid b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("wheezy64") @@ -457,6 +466,7 @@ Vagrant.configure(2) do |config| config.vm.define "darwin64" do |b| b.vm.box = "jhcook/yosemite-clitools" b.vm.provider :virtualbox do |v| + v.memory = 1536 + $wmem v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64'] v.customize ['modifyvm', :id, '--paravirtprovider', 'default'] # Adjust CPU settings according to @@ -482,7 +492,7 @@ Vagrant.configure(2) do |config| config.vm.define "freebsd64" do |b| b.vm.box = "freebsd/FreeBSD-10.3-RELEASE" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.ssh.shell = "sh" b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd @@ -498,8 +508,9 @@ Vagrant.configure(2) do |config| config.vm.define "openbsd64" do |b| b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end + b.ssh.shell = "sh" b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg(false) @@ -509,7 +520,7 @@ Vagrant.configure(2) do |config| config.vm.define "netbsd64" do |b| b.vm.box = "netbsd70-64" b.vm.provider :virtualbox do |v| - v.memory = 1536 + v.memory = 1024 + $wmem end b.vm.provision "packages netbsd", :type => :shell, :inline => packages_netbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("netbsd64") @@ -529,7 +540,7 @@ Vagrant.configure(2) do |config| b.ssh.insert_key = false b.vm.provider :virtualbox do |v| - v.memory = 2048 + v.memory = 1536 + $wmem #v.gui = true end diff --git a/docs/development.rst b/docs/development.rst index f8277aaa9..7d4c0774d 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -249,6 +249,8 @@ Usage:: # To create and provision the VM: vagrant up OS + # same, but use 6 VM cpus and 12 workers for pytest: + VMCPUS=6 XDISTN=12 vagrant up OS # To create an ssh session to the VM: vagrant ssh OS # To execute a command via ssh in the VM: