From a5a6ba0d77302f7c903f214b8048eaf56bb4def3 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 11 Oct 2015 16:07:11 +0200
Subject: [PATCH] integrate pytest-benchmark, test create, extract, list,
 delete, info, check, help, fixes #146

Instead of "realistic data", I chose the test data to be either all-zero (all-ascii-zero to be precise)
or all-random and benchmark them separately.
So we can better determine the cause (deduplication or storage) in case we see some performance regression.

"help" is benchmarked to see the minimum runtime when it basically does nothing.

also:
- refactor archiver execution core functionality into exec_cmd() so it can be used more flexibly
- tox: usually we want to skip benchmarks, only run them if requested manually
- install pytest-benchmark - run tox with "-r" to have it installed into your .tox envs
---
 borg/testsuite/archiver.py     |  57 +++++++++---------
 borg/testsuite/benchmark.py    | 105 +++++++++++++++++++++++++++++++++
 requirements.d/development.txt |   1 +
 tox.ini                        |   2 +-
 4 files changed, 136 insertions(+), 29 deletions(-)
 create mode 100644 borg/testsuite/benchmark.py

diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py
index 2ae565bb8..00a3a2bd9 100644
--- a/borg/testsuite/archiver.py
+++ b/borg/testsuite/archiver.py
@@ -71,6 +71,29 @@ class environment_variable:
                 os.environ[k] = v
 
 
+def exec_cmd(*args, archiver=None, fork=False, **kw):
+    if fork:
+        try:
+            borg = (sys.executable, '-m', 'borg.archiver')
+            output = subprocess.check_output(borg + args)
+            ret = 0
+        except subprocess.CalledProcessError as e:
+            output = e.output
+            ret = e.returncode
+        return ret, os.fsdecode(output)
+    else:
+        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
+        try:
+            sys.stdin = StringIO()
+            sys.stdout = sys.stderr = output = StringIO()
+            if archiver is None:
+                archiver = Archiver()
+            ret = archiver.run(list(args))
+            return ret, output.getvalue()
+        finally:
+            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+
+
 class ArchiverTestCaseBase(BaseTestCase):
 
     prefix = ''
@@ -102,34 +125,12 @@ class ArchiverTestCaseBase(BaseTestCase):
         shutil.rmtree(self.tmpdir)
 
     def cmd(self, *args, **kw):
-        exit_code = kw.get('exit_code', 0)
-        fork = kw.get('fork', False)
-        if fork:
-            try:
-                output = subprocess.check_output((sys.executable, '-m', 'borg.archiver') + args)
-                ret = 0
-            except subprocess.CalledProcessError as e:
-                output = e.output
-                ret = e.returncode
-            output = os.fsdecode(output)
-            if ret != exit_code:
-                print(output)
-            self.assert_equal(exit_code, ret)
-            return output
-        args = list(args)
-        stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr
-        try:
-            sys.stdin = StringIO()
-            output = StringIO()
-            sys.stdout = sys.stderr = output
-            ret = self.archiver.run(args)
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
-            if ret != exit_code:
-                print(output.getvalue())
-            self.assert_equal(exit_code, ret)
-            return output.getvalue()
-        finally:
-            sys.stdin, sys.stdout, sys.stderr = stdin, stdout, stderr
+        exit_code = kw.pop('exit_code', 0)
+        ret, output = exec_cmd(*args, archiver=self.archiver, **kw)
+        if ret != exit_code:
+            print(output)
+        self.assert_equal(ret, exit_code)
+        return output
 
     def create_src_archive(self, name):
         self.cmd('create', self.repository_location + '::' + name, src_dir)
diff --git a/borg/testsuite/benchmark.py b/borg/testsuite/benchmark.py
new file mode 100644
index 000000000..54d608a1a
--- /dev/null
+++ b/borg/testsuite/benchmark.py
@@ -0,0 +1,105 @@
+"""
+Do benchmarks using pytest-benchmark.
+
+Usage:
+
+    py.test --benchmark-only
+"""
+
+import os
+
+import pytest
+
+from .archiver import changedir, exec_cmd
+
+
+@pytest.fixture
+def cmd():
+    return exec_cmd
+
+
+@pytest.yield_fixture
+def repo_url(request, tmpdir):
+    os.environ['BORG_PASSPHRASE'] = '123456'
+    os.environ['BORG_CHECK_I_KNOW_WHAT_I_AM_DOING'] = '1'
+    os.environ['BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'] = '1'
+    os.environ['BORG_KEYS_DIR'] = str(tmpdir.join('keys'))
+    os.environ['BORG_CACHE_DIR'] = str(tmpdir.join('cache'))
+    yield str(tmpdir.join('repository'))
+    tmpdir.remove(rec=1)
+
+
+@pytest.fixture(params=["none", "passphrase"])
+def repo(request, cmd, repo_url):
+    cmd('init', '--encryption', request.param, repo_url)
+    return repo_url
+
+
+@pytest.yield_fixture(scope='session', params=["zeros", "random"])
+def testdata(request, tmpdir_factory):
+    count, size = 10, 1000*1000
+    p = tmpdir_factory.mktemp('data')
+    data_type = request.param
+    if data_type == 'zeros':
+        # note: do not use a binary zero (\0) to avoid sparse detection
+        data = lambda: b'0' * size
+    if data_type == 'random':
+        rnd = open('/dev/urandom', 'rb')
+        data = lambda: rnd.read(size)
+    for i in range(count):
+        with open(str(p.join(str(i))), "wb") as f:
+            f.write(data())
+    if data_type == 'random':
+        rnd.close()
+    yield str(p)
+    p.remove(rec=1)
+
+
+@pytest.fixture(params=['none', 'lz4'])
+def archive(request, cmd, repo, testdata):
+    archive_url = repo + '::test'
+    cmd('create', '--compression', request.param, archive_url, testdata)
+    return archive_url
+
+
+def test_create_none(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'none', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_create_lz4(benchmark, cmd, repo, testdata):
+    result, out = benchmark.pedantic(cmd, ('create', '--compression', 'lz4', repo + '::test', testdata))
+    assert result == 0
+
+
+def test_extract(benchmark, cmd, archive, tmpdir):
+    with changedir(str(tmpdir)):
+        result, out = benchmark.pedantic(cmd, ('extract', archive))
+    assert result == 0
+
+
+def test_delete(benchmark, cmd, archive):
+    result, out = benchmark.pedantic(cmd, ('delete', archive))
+    assert result == 0
+
+
+def test_list(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'list', archive)
+    assert result == 0
+
+
+def test_info(benchmark, cmd, archive):
+    result, out = benchmark(cmd, 'info', archive)
+    assert result == 0
+
+
+def test_check(benchmark, cmd, archive):
+    repo = archive.split('::')[0]
+    result, out = benchmark(cmd, 'check', repo)
+    assert result == 0
+
+
+def test_help(benchmark, cmd):
+    result, out = benchmark(cmd, 'help')
+    assert result == 0
+
diff --git a/requirements.d/development.txt b/requirements.d/development.txt
index 37677a00f..94f7273d4 100644
--- a/requirements.d/development.txt
+++ b/requirements.d/development.txt
@@ -2,4 +2,5 @@ tox
 mock
 pytest
 pytest-cov<2.0.0
+pytest-benchmark==3.0.0a3
 Cython
diff --git a/tox.ini b/tox.ini
index a9ccb5e04..c260b5063 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,6 +11,6 @@ changedir = {toxworkdir}
 deps =
      -rrequirements.d/development.txt
      attic
-commands = py.test --cov=borg --pyargs {posargs:borg.testsuite}
+commands = py.test --cov=borg --benchmark-skip --pyargs {posargs:borg.testsuite}
 # fakeroot -u needs some env vars:
 passenv = *