From 8317698f9bce2ff879acead9f5b02d5959ca8fa2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:22:24 +0100 Subject: [PATCH 1/7] hmac_sha256: replace own cython wrapper code by hmac.digest python stdlib (since py38) i measured performance of both: pretty much the same. --- src/borg/crypto/low_level.pyx | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index f726f3150..b43d4a79f 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -754,18 +754,7 @@ cdef class AES: def hmac_sha256(key, data): - cdef Py_buffer data_buf = ro_buffer(data) - cdef const unsigned char *key_ptr = key - cdef int key_len = len(key) - cdef unsigned char md[32] - try: - with nogil: - rc = HMAC(EVP_sha256(), key_ptr, key_len, data_buf.buf, data_buf.len, md, NULL) - if rc != md: - raise CryptoError('HMAC(EVP_sha256) failed') - finally: - PyBuffer_Release(&data_buf) - return PyBytes_FromStringAndSize( &md[0], 32) + return hmac.digest(key, data, 'sha256') def blake2b_256(key, data): From c63bd36a6c4b34b6b45f7b33e3f8c7d29bed51c0 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:29:42 +0100 Subject: [PATCH 2/7] hmac.digest: some more minor optimizations also some cosmetic changes: - import hmac module - use hmac.compare_digest --- src/borg/crypto/key.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 0cc487ab8..daed70009 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -1,5 +1,6 @@ import configparser import getpass +import hmac import os import shlex import sys @@ -7,7 +8,6 @@ import subprocess from binascii import a2b_base64, b2a_base64, hexlify from hashlib import sha256, sha512, pbkdf2_hmac -from hmac import HMAC, compare_digest from ..logger import create_logger @@ -193,7 +193,7 @@ def decrypt(self, id, data, decompress=True): def assert_id(self, id, data): if id: id_computed = self.id_hash(data) - if not compare_digest(id_computed, id): + if not hmac.compare_digest(id_computed, id): raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id)) def _tam_key(self, salt, context): @@ -213,7 +213,7 @@ def pack_and_authenticate_metadata(self, metadata_dict, context=b'manifest'): }) packed = msgpack.packb(metadata_dict) tam_key = self._tam_key(tam['salt'], context) - tam['hmac'] = HMAC(tam_key, packed, sha512).digest() + tam['hmac'] = hmac.digest(tam_key, packed, 'sha512') return msgpack.packb(metadata_dict) def unpack_and_verify_manifest(self, data, force_tam_not_required=False): @@ -252,8 +252,8 @@ def unpack_and_verify_manifest(self, data, force_tam_not_required=False): offset = data.index(tam_hmac) data[offset:offset + 64] = bytes(64) tam_key = self._tam_key(tam_salt, context=b'manifest') - calculated_hmac = HMAC(tam_key, data, sha512).digest() - if not compare_digest(calculated_hmac, tam_hmac): + calculated_hmac = hmac.digest(tam_key, data, 'sha512') + if not hmac.compare_digest(calculated_hmac, tam_hmac): raise TAMInvalid() logger.debug('TAM-verified manifest') return unpacked, True From 2566671d85ac5c2d4fefa60eaaf0461ccb10cdbd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:37:31 +0100 Subject: [PATCH 3/7] remove our hmac_sha256 tests as we're just calling python stdlib there, we can assume that they have tests for it. --- src/borg/testsuite/crypto.py | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 89d715bfe..3185c6941 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,7 +1,7 @@ from binascii import hexlify, unhexlify from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, \ - IntegrityError, blake2b_128, blake2b_256, hmac_sha256, is_libressl + IntegrityError, blake2b_128, blake2b_256, is_libressl from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes from ..crypto.low_level import hkdf_hmac_sha512 @@ -177,32 +177,6 @@ def test_AEAD(self): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) - def test_hmac_sha256(self): - # RFC 4231 test vectors - key = b'\x0b' * 20 - # Also test that this works with memory views - data = memoryview(unhexlify('4869205468657265')) - hmac = unhexlify('b0344c61d8db38535ca8afceaf0bf12b' - '881dc200c9833da726e9376c2e32cff7') - assert hmac_sha256(key, data) == hmac - key = unhexlify('4a656665') - data = unhexlify('7768617420646f2079612077616e7420' - '666f72206e6f7468696e673f') - hmac = unhexlify('5bdcc146bf60754e6a042426089575c7' - '5a003f089d2739839dec58b964ec3843') - assert hmac_sha256(key, data) == hmac - key = b'\xaa' * 20 - data = b'\xdd' * 50 - hmac = unhexlify('773ea91e36800e46854db8ebd09181a7' - '2959098b3ef8c122d9635514ced565fe') - assert hmac_sha256(key, data) == hmac - key = unhexlify('0102030405060708090a0b0c0d0e0f10' - '111213141516171819') - data = b'\xcd' * 50 - hmac = unhexlify('82558a389a443c0ea4cc819899f2083a' - '85f0faa3e578f8077a2e3ff46729665b') - assert hmac_sha256(key, data) == hmac - def test_blake2b_256(self): # In BLAKE2 the output length actually is part of the hashes personality - it is *not* simple truncation like in # the SHA-2 family. Therefore we need to generate test vectors ourselves (as is true for most applications that From 07159342ba636aa5843e33719bb2b516c9a81551 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:47:40 +0100 Subject: [PATCH 4/7] remove our blake2b_128/256 tests as we're just calling python stdlib there, we can assume that they have tests for it. --- src/borg/testsuite/crypto.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 3185c6941..7206d4385 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,7 +1,7 @@ from binascii import hexlify, unhexlify from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, \ - IntegrityError, blake2b_128, blake2b_256, is_libressl + IntegrityError, is_libressl from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes from ..crypto.low_level import hkdf_hmac_sha512 @@ -177,28 +177,6 @@ def test_AEAD(self): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted)) - def test_blake2b_256(self): - # In BLAKE2 the output length actually is part of the hashes personality - it is *not* simple truncation like in - # the SHA-2 family. Therefore we need to generate test vectors ourselves (as is true for most applications that - # are not precisely vanilla BLAKE2b-512 or BLAKE2s-256). - # - # Obtained via "b2sum" utility from the official BLAKE2 repository. It calculates the exact hash of a file's - # contents, no extras (like length) included. - assert blake2b_256(b'', b'abc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') - assert blake2b_256(b'a', b'bc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') - assert blake2b_256(b'ab', b'c') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') - assert blake2b_256(b'abc', b'') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') - - key = unhexlify('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676') - data = memoryview(b'1234567890' * 100) - assert blake2b_256(key, data) == unhexlify('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2') - - def test_blake2b_128(self): - # (see above) - assert blake2b_128(b'') == unhexlify('cae66941d9efbd404e4d88758ea67670') - assert blake2b_128(b'abc') == unhexlify('cf4ab791c62b8d2b2109c90275287816') - assert blake2b_128(b'abcd'*8) == unhexlify('0f759d9a32d3f99250c1781a8baa58b9') - # These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/ # who claims to have verified these against independent Python and C++ implementations. From c5540229c45795db4ac67db152db36695ca6b4f4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:51:50 +0100 Subject: [PATCH 5/7] remove the blake2b compat checker script this was used to compare compatibility of our vendored blake2b code (which we do not have any more) against the python stdlib blake2b code (which we always use now anyway). --- scripts/py36-blake2.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 scripts/py36-blake2.py diff --git a/scripts/py36-blake2.py b/scripts/py36-blake2.py deleted file mode 100644 index fff13b588..000000000 --- a/scripts/py36-blake2.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -This script checks compatibility of crypto.blake2b_256 against hashlib.blake2b in CPython 3.6. -""" - -import hashlib -import sys - - -def test_b2(b2_input, b2_output): - digest = hashlib.blake2b(b2_input, digest_size=32).digest() - identical = b2_output == digest - - print('Input: ', b2_input.hex()) - print('Expected: ', b2_output.hex()) - print('Calculated:', digest.hex()) - print('Identical: ', identical) - print() - if not identical: - sys.exit(1) - - -test_b2( - bytes.fromhex('037fb9b75b20d623f1d5a568050fccde4a1b7c5f5047432925e941a17c7a2d0d7061796c6f6164'), - bytes.fromhex('a22d4fc81bb61c3846c334a09eaf28d22dd7df08c9a7a41e713ef28d80eebd45') -) - -test_b2( - b'abc', - bytes.fromhex('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') -) - -test_b2( - bytes.fromhex('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676') + b'1234567890' * 100, - bytes.fromhex('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2'), -) From 41536f9f79da7d7bc3d1fa3248a1b7f26006be6f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 21:55:23 +0100 Subject: [PATCH 6/7] fixup: remove now unused import --- src/borg/testsuite/crypto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 7206d4385..6420f36c8 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,4 +1,4 @@ -from binascii import hexlify, unhexlify +from binascii import hexlify from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, \ IntegrityError, is_libressl From 862eefb96d8909802892ae29a2569b228f889849 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 5 Mar 2022 22:26:04 +0100 Subject: [PATCH 7/7] fixup: fix SELFTEST_COUNT --- src/borg/selftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/selftest.py b/src/borg/selftest.py index 1f374679e..dfe7abb54 100644 --- a/src/borg/selftest.py +++ b/src/borg/selftest.py @@ -29,7 +29,7 @@ ChunkerTestCase, ] -SELFTEST_COUNT = 38 +SELFTEST_COUNT = 35 class SelfTestResult(TestResult):