Merge pull request #970 from enkore/feature/selftest

Add power-on self-test

borg/archiver.py

@@ -34,6 +34,7 @@ from .constants import *  # NOQA
 from .key import key_creator, RepoKey, PassphraseKey
 from .archive import Archive, ArchiveChecker, ArchiveRecreater
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
+from .selftest import selftest
 from .hashindex import ChunkIndexEntry
 
 has_lchflags = hasattr(os, 'lchflags')
@@ -1924,13 +1925,17 @@ class Archiver:
         update_excludes(args)
         return args
 
+    def prerun_checks(self, logger):
+        check_extension_modules()
+        selftest(logger)
+
     def run(self, args):
         os.umask(args.umask)  # early, before opening files
         self.lock_wait = args.lock_wait
         setup_logging(level=args.log_level, is_serve=args.func == self.do_serve)  # do not use loggers before this!
         if args.show_version:
             logger.info('borgbackup version %s' % __version__)
-        check_extension_modules()
+        self.prerun_checks(logger)
         if is_slow_msgpack():
             logger.warning("Using a pure-python msgpack! This will result in lower performance.")
         return args.func(args)

borg/crypto.pyx

@@ -5,7 +5,7 @@ This could be replaced by PyCrypto maybe?
 from libc.stdlib cimport malloc, free
 from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 
-API_VERSION = 2
+API_VERSION = 3
 
 
 cdef extern from "openssl/rand.h":

borg/helpers.py

@@ -80,7 +80,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if chunker.API_VERSION != 2:
         raise ExtensionModuleError
-    if crypto.API_VERSION != 2:
+    if crypto.API_VERSION != 3:
         raise ExtensionModuleError
     if platform.API_VERSION != 2:
         raise ExtensionModuleError

borg/selftest.py

@@ -0,0 +1,79 @@
+"""
+Self testing module
+===================
+
+The selftest() function runs a small test suite of relatively fast tests that are meant to discover issues
+with the way Borg was compiled or packaged and also bugs in Borg itself.
+
+Theses tests are a subset of the borg/testsuite and are run with Pythons built-in unittest, hence none of
+the tests used for this can or should be ported to py.test currently.
+
+To assert that self test discovery works correctly the number of tests is kept in the SELFTEST_COUNT
+variable. SELFTEST_COUNT must be updated if new tests are added or removed to or from any of the tests
+used here.
+"""
+
+
+import sys
+import time
+from unittest import TestResult, TestSuite, defaultTestLoader
+
+from .testsuite.hashindex import HashIndexDataTestCase, HashIndexRefcountingTestCase, HashIndexTestCase
+from .testsuite.crypto import CryptoTestCase
+from .testsuite.chunker import ChunkerTestCase
+
+SELFTEST_CASES = [
+    HashIndexDataTestCase,
+    HashIndexRefcountingTestCase,
+    HashIndexTestCase,
+    CryptoTestCase,
+    ChunkerTestCase,
+]
+
+SELFTEST_COUNT = 27
+
+
+class SelfTestResult(TestResult):
+    def __init__(self):
+        super().__init__()
+        self.successes = []
+
+    def addSuccess(self, test):
+        super().addSuccess(test)
+        self.successes.append(test)
+
+    def test_name(self, test):
+        return test.shortDescription() or str(test)
+
+    def log_results(self, logger):
+        for test, failure in self.errors + self.failures + self.unexpectedSuccesses:
+            logger.error('self test %s FAILED:\n%s', self.test_name(test), failure)
+        for test, reason in self.skipped:
+            logger.warning('self test %s skipped: %s', self.test_name(test), reason)
+
+    def successful_test_count(self):
+        return len(self.successes)
+
+
+def selftest(logger):
+    selftest_started = time.perf_counter()
+    result = SelfTestResult()
+    test_suite = TestSuite()
+    for test_case in SELFTEST_CASES:
+        test_suite.addTest(defaultTestLoader.loadTestsFromTestCase(test_case))
+    test_suite.run(result)
+    result.log_results(logger)
+    successful_tests = result.successful_test_count()
+    count_mismatch = successful_tests != SELFTEST_COUNT
+    if result.wasSuccessful() and count_mismatch:
+        # only print this if all tests succeeded
+        logger.error("self test count (%d != %d) mismatch, either test discovery is broken or a test was added "
+                     "without updating borg.selftest",
+                     successful_tests, SELFTEST_COUNT)
+    if not result.wasSuccessful() or count_mismatch:
+        logger.error("self test failed\n"
+                     "This is a bug either in Borg or in the package / distribution you use.")
+        sys.exit(2)
+        assert False, "sanity assertion failed: ran beyond sys.exit()"
+    selftest_elapsed = time.perf_counter() - selftest_started
+    logger.debug("%d self tests completed in %.2f seconds", successful_tests, selftest_elapsed)

borg/testsuite/__init__.py

@@ -8,7 +8,8 @@ import sysconfig
 import time
 import unittest
 from ..xattr import get_all
-from ..logger import setup_logging
+
+# Note: this is used by borg.selftest, do not use or import py.test functionality here.
 
 try:
     import llfuse
@@ -17,6 +18,11 @@ try:
 except ImportError:
     have_fuse_mtime_ns = False
 
+try:
+    from pytest import raises
+except ImportError:
+    raises = None
+
 has_lchflags = hasattr(os, 'lchflags')
 
 
@@ -31,9 +37,6 @@ else:
 if sys.platform.startswith('netbsd'):
     st_mtime_ns_round = -4  # only >1 microsecond resolution here?
 
-# Ensure that the loggers exist for all tests
-setup_logging()
-
 
 class BaseTestCase(unittest.TestCase):
     """
@@ -42,9 +45,13 @@ class BaseTestCase(unittest.TestCase):
     assert_not_in = unittest.TestCase.assertNotIn
     assert_equal = unittest.TestCase.assertEqual
     assert_not_equal = unittest.TestCase.assertNotEqual
-    assert_raises = unittest.TestCase.assertRaises
     assert_true = unittest.TestCase.assertTrue
 
+    if raises:
+        assert_raises = staticmethod(raises)
+    else:
+        assert_raises = unittest.TestCase.assertRaises
+
     @contextmanager
     def assert_creates_file(self, path):
         self.assert_true(not os.path.exists(path), '{} should not exist'.format(path))

borg/testsuite/archiver.py

@@ -61,6 +61,7 @@ def exec_cmd(*args, archiver=None, fork=False, exe=None, **kw):
             sys.stdout = sys.stderr = output = StringIO()
             if archiver is None:
                 archiver = Archiver()
+            archiver.prerun_checks = lambda *args: None
             archiver.exit_code = EXIT_SUCCESS
             args = archiver.parse_args(list(args))
             ret = archiver.run(args)

borg/testsuite/chunker.py

@@ -4,6 +4,9 @@ from ..chunker import Chunker, buzhash, buzhash_update
 from ..constants import *  # NOQA
 from . import BaseTestCase
 
+# Note: these tests are part of the self test, do not use or import py.test functionality here.
+#       See borg.selftest for details. If you add/remove test methods, update SELFTEST_COUNT
+
 
 class ChunkerTestCase(BaseTestCase):
 

borg/testsuite/conftest.py

@@ -0,0 +1,4 @@
+from ..logger import setup_logging
+
+# Ensure that the loggers exist for all tests
+setup_logging()

borg/testsuite/crypto.py

@@ -3,6 +3,9 @@ from binascii import hexlify, unhexlify
 from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256
 from . import BaseTestCase
 
+# Note: these tests are part of the self test, do not use or import py.test functionality here.
+#       See borg.selftest for details. If you add/remove test methods, update SELFTEST_COUNT
+
 
 class CryptoTestCase(BaseTestCase):
 

borg/testsuite/hashindex.py

@@ -1,15 +1,16 @@
 import base64
 import hashlib
 import os
-import struct
 import tempfile
 import zlib
 
-import pytest
 from ..hashindex import NSIndex, ChunkIndex
 from .. import hashindex
 from . import BaseTestCase
 
+# Note: these tests are part of the self test, do not use or import py.test functionality here.
+#       See borg.selftest for details. If you add/remove test methods, update SELFTEST_COUNT
+
 
 def H(x):
     # make some 32byte long thing that depends on x
@@ -194,7 +195,7 @@ class HashIndexRefcountingTestCase(BaseTestCase):
     def test_decref_zero(self):
         idx1 = ChunkIndex()
         idx1[H(1)] = 0, 0, 0
-        with pytest.raises(AssertionError):
+        with self.assert_raises(AssertionError):
             idx1.decref(H(1))
 
     def test_incref_decref(self):
@@ -208,18 +209,18 @@ class HashIndexRefcountingTestCase(BaseTestCase):
 
     def test_setitem_raises(self):
         idx1 = ChunkIndex()
-        with pytest.raises(AssertionError):
+        with self.assert_raises(AssertionError):
             idx1[H(1)] = hashindex.MAX_VALUE + 1, 0, 0
 
     def test_keyerror(self):
         idx = ChunkIndex()
-        with pytest.raises(KeyError):
+        with self.assert_raises(KeyError):
             idx.incref(H(1))
-        with pytest.raises(KeyError):
+        with self.assert_raises(KeyError):
             idx.decref(H(1))
-        with pytest.raises(KeyError):
+        with self.assert_raises(KeyError):
             idx[H(1)]
-        with pytest.raises(OverflowError):
+        with self.assert_raises(OverflowError):
             idx.add(H(1), -1, 0, 0)
 
 
@@ -269,10 +270,11 @@ class HashIndexDataTestCase(BaseTestCase):
         assert idx1[H(3)] == (hashindex.MAX_VALUE, 6, 7)
 
 
-def test_nsindex_segment_limit():
-    idx = NSIndex()
-    with pytest.raises(AssertionError):
-        idx[H(1)] = hashindex.MAX_VALUE + 1, 0
-    assert H(1) not in idx
-    idx[H(2)] = hashindex.MAX_VALUE, 0
-    assert H(2) in idx
+class NSIndexTestCase(BaseTestCase):
+    def test_nsindex_segment_limit(self):
+        idx = NSIndex()
+        with self.assert_raises(AssertionError):
+            idx[H(1)] = hashindex.MAX_VALUE + 1, 0
+        assert H(1) not in idx
+        idx[H(2)] = hashindex.MAX_VALUE, 0
+        assert H(2) in idx