Merge pull request #6408 from ThomasWaldmann/hmac-digest-master

hmac and blake2b minor optimizations and cleanups

scripts/py36-blake2.py

@@ -1,35 +0,0 @@
-"""
-This script checks compatibility of crypto.blake2b_256 against hashlib.blake2b in CPython 3.6.
-"""
-
-import hashlib
-import sys
-
-
-def test_b2(b2_input, b2_output):
-    digest = hashlib.blake2b(b2_input, digest_size=32).digest()
-    identical = b2_output == digest
-
-    print('Input:     ', b2_input.hex())
-    print('Expected:  ', b2_output.hex())
-    print('Calculated:', digest.hex())
-    print('Identical: ', identical)
-    print()
-    if not identical:
-        sys.exit(1)
-
-
-test_b2(
-    bytes.fromhex('037fb9b75b20d623f1d5a568050fccde4a1b7c5f5047432925e941a17c7a2d0d7061796c6f6164'),
-    bytes.fromhex('a22d4fc81bb61c3846c334a09eaf28d22dd7df08c9a7a41e713ef28d80eebd45')
-)
-
-test_b2(
-    b'abc',
-    bytes.fromhex('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319')
-)
-
-test_b2(
-    bytes.fromhex('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676') + b'1234567890' * 100,
-    bytes.fromhex('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2'),
-)

src/borg/crypto/key.py

@@ -1,5 +1,6 @@
 import configparser
 import getpass
+import hmac
 import os
 import shlex
 import sys
@@ -7,7 +8,6 @@
 import subprocess
 from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, sha512, pbkdf2_hmac
-from hmac import HMAC, compare_digest
 
 from ..logger import create_logger
 
@@ -193,7 +193,7 @@ def decrypt(self, id, data, decompress=True):
     def assert_id(self, id, data):
         if id:
             id_computed = self.id_hash(data)
-            if not compare_digest(id_computed, id):
+            if not hmac.compare_digest(id_computed, id):
                 raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id))
 
     def _tam_key(self, salt, context):
@@ -213,7 +213,7 @@ def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'):
         })
         packed = msgpack.packb(metadata_dict)
         tam_key = self._tam_key(tam['salt'], context)
-        tam['hmac'] = HMAC(tam_key, packed, sha512).digest()
+        tam['hmac'] = hmac.digest(tam_key, packed, 'sha512')
         return msgpack.packb(metadata_dict)
 
     def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
@@ -252,8 +252,8 @@ def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
         offset = data.index(tam_hmac)
         data[offset:offset + 64] = bytes(64)
         tam_key = self._tam_key(tam_salt, context=b'manifest')
-        calculated_hmac = HMAC(tam_key, data, sha512).digest()
+        calculated_hmac = hmac.digest(tam_key, data, 'sha512')
-        if not compare_digest(calculated_hmac, tam_hmac):
+        if not hmac.compare_digest(calculated_hmac, tam_hmac):
             raise TAMInvalid()
         logger.debug('TAM-verified manifest')
         return unpacked, True

src/borg/crypto/low_level.pyx

@@ -754,18 +754,7 @@ cdef class AES:
 
 
 def hmac_sha256(key, data):
-    cdef Py_buffer data_buf = ro_buffer(data)
+    return hmac.digest(key, data, 'sha256')
-    cdef const unsigned char *key_ptr = key
-    cdef int key_len = len(key)
-    cdef unsigned char md[32]
-    try:
-        with nogil:
-            rc = HMAC(EVP_sha256(), key_ptr, key_len, <const unsigned char*> data_buf.buf, data_buf.len, md, NULL)
-        if rc != md:
-            raise CryptoError('HMAC(EVP_sha256) failed')
-    finally:
-        PyBuffer_Release(&data_buf)
-    return PyBytes_FromStringAndSize(<char*> &md[0], 32)
 
 
 def blake2b_256(key, data):

src/borg/selftest.py

@@ -29,7 +29,7 @@
     ChunkerTestCase,
 ]
 
-SELFTEST_COUNT = 38
+SELFTEST_COUNT = 35
 
 
 class SelfTestResult(TestResult):

src/borg/testsuite/crypto.py

@@ -1,7 +1,7 @@
-from binascii import hexlify, unhexlify
+from binascii import hexlify
 
 from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, \
-                               IntegrityError, blake2b_128, blake2b_256, hmac_sha256, is_libressl
+                               IntegrityError, is_libressl
 from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes
 from ..crypto.low_level import hkdf_hmac_sha512
 
@@ -177,54 +177,6 @@ def test_AEAD(self):
             self.assert_raises(IntegrityError,
                                lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted))
 
-    def test_hmac_sha256(self):
-        # RFC 4231 test vectors
-        key = b'\x0b' * 20
-        # Also test that this works with memory views
-        data = memoryview(unhexlify('4869205468657265'))
-        hmac = unhexlify('b0344c61d8db38535ca8afceaf0bf12b'
-                         '881dc200c9833da726e9376c2e32cff7')
-        assert hmac_sha256(key, data) == hmac
-        key = unhexlify('4a656665')
-        data = unhexlify('7768617420646f2079612077616e7420'
-                         '666f72206e6f7468696e673f')
-        hmac = unhexlify('5bdcc146bf60754e6a042426089575c7'
-                         '5a003f089d2739839dec58b964ec3843')
-        assert hmac_sha256(key, data) == hmac
-        key = b'\xaa' * 20
-        data = b'\xdd' * 50
-        hmac = unhexlify('773ea91e36800e46854db8ebd09181a7'
-                         '2959098b3ef8c122d9635514ced565fe')
-        assert hmac_sha256(key, data) == hmac
-        key = unhexlify('0102030405060708090a0b0c0d0e0f10'
-                        '111213141516171819')
-        data = b'\xcd' * 50
-        hmac = unhexlify('82558a389a443c0ea4cc819899f2083a'
-                         '85f0faa3e578f8077a2e3ff46729665b')
-        assert hmac_sha256(key, data) == hmac
-
-    def test_blake2b_256(self):
-        # In BLAKE2 the output length actually is part of the hashes personality - it is *not* simple truncation like in
-        # the SHA-2 family. Therefore we need to generate test vectors ourselves (as is true for most applications that
-        # are not precisely vanilla BLAKE2b-512 or BLAKE2s-256).
-        #
-        # Obtained via "b2sum" utility from the official BLAKE2 repository. It calculates the exact hash of a file's
-        # contents, no extras (like length) included.
-        assert blake2b_256(b'', b'abc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319')
-        assert blake2b_256(b'a', b'bc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319')
-        assert blake2b_256(b'ab', b'c') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319')
-        assert blake2b_256(b'abc', b'') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319')
-
-        key = unhexlify('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676')
-        data = memoryview(b'1234567890' * 100)
-        assert blake2b_256(key, data) == unhexlify('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2')
-
-    def test_blake2b_128(self):
-        # (see above)
-        assert blake2b_128(b'') == unhexlify('cae66941d9efbd404e4d88758ea67670')
-        assert blake2b_128(b'abc') == unhexlify('cf4ab791c62b8d2b2109c90275287816')
-        assert blake2b_128(b'abcd'*8) == unhexlify('0f759d9a32d3f99250c1781a8baa58b9')
-
     # These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/
     # who claims to have verified these against independent Python and C++ implementations.