integrate pytest-benchmark, test create, extract, list, delete, info, check, help, fixes #146

Instead of "realistic data", I chose the test data to be either all-zero (all-ascii-zero to be precise)
or all-random and benchmark them separately.
So we can better determine the cause (deduplication or storage) in case we see some performance regression.

"help" is benchmarked to see the minimum runtime when it basically does nothing.

also:
- refactor archiver execution core functionality into exec_cmd() so it can be used more flexibly
- tox: usually we want to skip benchmarks, only run them if requested manually
- install pytest-benchmark - run tox with "-r" to have it installed into your .tox envs

borg/testsuite/archiver.py

@@ -71,6 +71,29 @@ class environment_variable:
                 os.environ[k] = v
 
 
+def exec_cmd(*args, archiver=None, fork=False, **kw):
+    if fork:
+        try:
+            borg = (sys.executable, '-m', 'borg.archiver')
+            output = subprocess.check_output(borg + args)
+            ret = 0
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            ret = e.returncode
+        return ret, os.fsdecode(output)
+    else:
+        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
+        try:
+            sys.stdin = StringIO()
+            sys.stdout = sys.stderr = output = StringIO()
+            if archiver is None:
+                archiver = Archiver()
+            ret = archiver.run(list(args))
+            return ret, output.getvalue()
+        finally:
+            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+
+
 class ArchiverTestCaseBase(BaseTestCase):
 
     prefix = ''
@@ -102,34 +125,12 @@ class ArchiverTestCaseBase(BaseTestCase):
         shutil.rmtree(self.tmpdir)
 
     def cmd(self, *args, **kw):
-        exit_code = kw.get('exit_code', 0)
-        fork = kw.get('fork', False)
-        if fork:
-            try:
-                output = subprocess.check_output((sys.executable, '-m', 'borg.archiver') + args)
-                ret = 0
-            except subprocess.CalledProcessError as e:
-                output = e.output
-                ret = e.returncode
-            output = os.fsdecode(output)
-            if ret != exit_code:
-                print(output)
-            self.assert_equal(exit_code, ret)
-            return output
-        args = list(args)
-        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
-        try:
-            sys.stdin = StringIO()
-            output = StringIO()
-            sys.stdout = sys.stderr = output
-            ret = self.archiver.run(args)
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
-            if ret != exit_code:
-                print(output.getvalue())
-            self.assert_equal(exit_code, ret)
-            return output.getvalue()
-        finally:
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+        exit_code = kw.pop('exit_code', 0)
+        ret, output = exec_cmd(*args, archiver=self.archiver, **kw)
+        if ret != exit_code:
+            print(output)
+        self.assert_equal(ret, exit_code)
+        return output
 
     def create_src_archive(self, name):
         self.cmd('create', self.repository_location + '::' + name, src_dir)

borg/testsuite/benchmark.py

@@ -0,0 +1,105 @@
+"""
+Do benchmarks using pytest-benchmark.
+
+Usage:
+
+    py.test --benchmark-only
+"""
+
+import os
+
+import pytest
+
+from .archiver import changedir, exec_cmd
+
+
+@pytest.fixture
+def cmd():
+    return exec_cmd
+
+
+@pytest.yield_fixture
+def repo_url(request, tmpdir):
+    os.environ['BORG_PASSPHRASE'] = '123456'
+    os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
+    os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1'
+    os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys'))
+    os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache'))
+    yield str(tmpdir.join('repository'))
+    tmpdir.remove(rec=1)
+
+
+@pytest.fixture(params=["none", "passphrase"])
+def repo(request, cmd, repo_url):
+    cmd('init', '--encryption', request.param, repo_url)
+    return repo_url
+
+
+@pytest.yield_fixture(scope='session', params=["zeros", "random"])
+def testdata(request, tmpdir_factory):
+    count, size = 10, 1000*1000
+    p = tmpdir_factory.mktemp('data')
+    data_type = request.param
+    if data_type == 'zeros':
+        # note: do not use a binary zero (\0) to avoid sparse detection
+        data = lambda: b'0' * size
+    if data_type == 'random':
+        rnd = open('/dev/urandom', 'rb')
+        data = lambda: rnd.read(size)
+    for i in range(count):
+        with open(str(p.join(str(i))), "wb") as f:
+            f.write(data())
+    if data_type == 'random':
+        rnd.close()
+    yield str(p)
+    p.remove(rec=1)
+
+
+@pytest.fixture(params=['none', 'lz4'])
+def archive(request, cmd, repo, testdata):
+    archive_url = repo + '::test'
+    cmd('create', '--compression', request.param, archive_url, testdata)
+    return archive_url
+
+
+def test_create_none(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'none', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_create_lz4(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'lz4', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_extract(benchmark, cmd, archive, tmpdir):
+    with changedir(str(tmpdir)):
+        result, out = benchmark.pedantic(cmd, ('extract', archive))
+    assert result == 0
+
+
+def test_delete(benchmark, cmd, archive):
+    result, out = benchmark.pedantic(cmd, ('delete', archive))
+    assert result == 0
+
+
+def test_list(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'list', archive)
+    assert result == 0
+
+
+def test_info(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'info', archive)
+    assert result == 0
+
+
+def test_check(benchmark, cmd, archive):
+    repo = archive.split('::')[0]
+    result, out = benchmark(cmd, 'check', repo)
+    assert result == 0
+
+
+def test_help(benchmark, cmd):
+    result, out = benchmark(cmd, 'help')
+    assert result == 0
+

requirements.d/development.txt

@@ -2,4 +2,5 @@ tox
 mock
 pytest
 pytest-cov<2.0.0
+pytest-benchmark==3.0.0a3
 Cython

tox.ini

@@ -11,6 +11,6 @@ changedir = {toxworkdir}
 deps =
      -rrequirements.d/development.txt
      attic
-commands = py.test --cov=borg --pyargs {posargs:borg.testsuite}
+commands = py.test --cov=borg --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *