From a19c9861bfad8b75fd3261046e5b9d9c27f8abcf Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 20 Jun 2016 23:40:30 +0200 Subject: [PATCH 1/3] crypto: use pointer to cipher context this is one of the steps needed to make borg compatible to openssl 1.1. --- borg/crypto.pyx | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 172fe0745..0d408bb7f 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -21,8 +21,8 @@ cdef extern from "openssl/evp.h": ctypedef struct ENGINE: pass const EVP_CIPHER *EVP_aes_256_ctr() - void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a) - void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a) + EVP_CIPHER_CTX *EVP_CIPHER_CTX_new() + void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a) int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv) @@ -56,24 +56,24 @@ def num_aes_blocks(int length): cdef class AES: """A thin wrapper around the OpenSSL EVP cipher API """ - cdef EVP_CIPHER_CTX ctx + cdef EVP_CIPHER_CTX *ctx cdef int is_encrypt def __cinit__(self, is_encrypt, key, iv=None): - EVP_CIPHER_CTX_init(&self.ctx) + self.ctx = EVP_CIPHER_CTX_new() self.is_encrypt = is_encrypt # Set cipher type and mode cipher_mode = EVP_aes_256_ctr() if self.is_encrypt: - if not EVP_EncryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL): + if not EVP_EncryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): raise Exception('EVP_EncryptInit_ex failed') else: # decrypt - if not EVP_DecryptInit_ex(&self.ctx, cipher_mode, NULL, NULL, NULL): + if not EVP_DecryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): raise Exception('EVP_DecryptInit_ex failed') self.reset(key, iv) def __dealloc__(self): - EVP_CIPHER_CTX_cleanup(&self.ctx) + EVP_CIPHER_CTX_free(self.ctx) def reset(self, key=None, iv=None): cdef const unsigned char *key2 = NULL @@ -84,10 +84,10 @@ cdef class AES: iv2 = iv # Initialise key and IV if self.is_encrypt: - if not EVP_EncryptInit_ex(&self.ctx, NULL, NULL, key2, iv2): + if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2): raise Exception('EVP_EncryptInit_ex failed') else: # decrypt - if not EVP_DecryptInit_ex(&self.ctx, NULL, NULL, key2, iv2): + if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, key2, iv2): raise Exception('EVP_DecryptInit_ex failed') @property @@ -103,10 +103,10 @@ cdef class AES: if not out: raise MemoryError try: - if not EVP_EncryptUpdate(&self.ctx, out, &outl, data, inl): + if not EVP_EncryptUpdate(self.ctx, out, &outl, data, inl): raise Exception('EVP_EncryptUpdate failed') ctl = outl - if not EVP_EncryptFinal_ex(&self.ctx, out+ctl, &outl): + if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl): raise Exception('EVP_EncryptFinal failed') ctl += outl return out[:ctl] @@ -124,10 +124,10 @@ cdef class AES: if not out: raise MemoryError try: - if not EVP_DecryptUpdate(&self.ctx, out, &outl, data, inl): + if not EVP_DecryptUpdate(self.ctx, out, &outl, data, inl): raise Exception('EVP_DecryptUpdate failed') ptl = outl - if EVP_DecryptFinal_ex(&self.ctx, out+ptl, &outl) <= 0: + if EVP_DecryptFinal_ex(self.ctx, out+ptl, &outl) <= 0: # this error check is very important for modes with padding or # authentication. for them, a failure here means corrupted data. # CTR mode does not use padding nor authentication. From 27c0d0f0743c933b5c6efde72cffc0e8f0f05dcd Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 20:06:36 +0200 Subject: [PATCH 2/3] move attic test dependency into own file so you can just empty that file to remove the attic-based tests when testing in a OpenSSL 1.1 environment. --- requirements.d/attic.txt | 5 +++++ tox.ini | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 requirements.d/attic.txt diff --git a/requirements.d/attic.txt b/requirements.d/attic.txt new file mode 100644 index 000000000..b5068ffd4 --- /dev/null +++ b/requirements.d/attic.txt @@ -0,0 +1,5 @@ +# Please note: +# attic only builds using OpenSSL 1.0.x, it can not be installed using OpenSSL >= 1.1.0. +# If attic is not installed, our unit tests will just skip the tests that require attic. +attic + diff --git a/tox.ini b/tox.ini index 0473cb271..c7b49bcaa 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ envlist = py{34,35},flake8 changedir = {toxworkdir} deps = -rrequirements.d/development.txt - attic + -rrequirements.d/attic.txt commands = py.test --cov=borg --cov-config=../.coveragerc --benchmark-skip --pyargs {posargs:borg.testsuite} # fakeroot -u needs some env vars: passenv = * From b5362fa5c88fb24d039b7d4293d63824189864c3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 21 Jun 2016 20:20:48 +0200 Subject: [PATCH 3/3] make borg build/work on OpenSSL 1.0 and 1.1, fixes #1187 in openssl 1.1, the cipher context is opaque, members can not be accessed directly. we only used this for ctx.iv to determine the current IV (counter value). now, we just remember the original IV, count the AES blocks we process and then compute iv = iv_orig + blocks. that way, it works on OpenSSL 1.0.x and >= 1.1 in the same way. --- borg/crypto.pyx | 39 +++++++++++++++++++++++++++++++++++++-- borg/testsuite/crypto.py | 22 ++++++++++++++++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/borg/crypto.pyx b/borg/crypto.pyx index 0d408bb7f..556f0fbce 100644 --- a/borg/crypto.pyx +++ b/borg/crypto.pyx @@ -16,7 +16,6 @@ cdef extern from "openssl/evp.h": ctypedef struct EVP_CIPHER: pass ctypedef struct EVP_CIPHER_CTX: - unsigned char *iv pass ctypedef struct ENGINE: pass @@ -40,12 +39,40 @@ import struct _int = struct.Struct('>I') _long = struct.Struct('>Q') +_2long = struct.Struct('>QQ') bytes_to_int = lambda x, offset=0: _int.unpack_from(x, offset)[0] bytes_to_long = lambda x, offset=0: _long.unpack_from(x, offset)[0] long_to_bytes = lambda x: _long.pack(x) +def bytes16_to_int(b, offset=0): + h, l = _2long.unpack_from(b, offset) + return (h << 64) + l + + +def int_to_bytes16(i): + max_uint64 = 0xffffffffffffffff + l = i & max_uint64 + h = (i >> 64) & max_uint64 + return _2long.pack(h, l) + + +def increment_iv(iv, amount=1): + """ + Increment the IV by the given amount (default 1). + + :param iv: input IV, 16 bytes (128 bit) + :param amount: increment value + :return: input_IV + amount, 16 bytes (128 bit) + """ + assert len(iv) == 16 + iv = bytes16_to_int(iv) + iv += amount + iv = int_to_bytes16(iv) + return iv + + def num_aes_blocks(int length): """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data. Note: this is only correct for modes without padding, like AES-CTR. @@ -58,6 +85,8 @@ cdef class AES: """ cdef EVP_CIPHER_CTX *ctx cdef int is_encrypt + cdef unsigned char iv_orig[16] + cdef int blocks def __cinit__(self, is_encrypt, key, iv=None): self.ctx = EVP_CIPHER_CTX_new() @@ -82,6 +111,10 @@ cdef class AES: key2 = key if iv: iv2 = iv + assert isinstance(iv, bytes) and len(iv) == 16 + for i in range(16): + self.iv_orig[i] = iv[i] + self.blocks = 0 # number of AES blocks encrypted starting with iv_orig # Initialise key and IV if self.is_encrypt: if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2): @@ -92,7 +125,7 @@ cdef class AES: @property def iv(self): - return self.ctx.iv[:16] + return increment_iv(self.iv_orig[:16], self.blocks) def encrypt(self, data): cdef int inl = len(data) @@ -109,6 +142,7 @@ cdef class AES: if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl): raise Exception('EVP_EncryptFinal failed') ctl += outl + self.blocks += num_aes_blocks(ctl) return out[:ctl] finally: free(out) @@ -133,6 +167,7 @@ cdef class AES: # CTR mode does not use padding nor authentication. raise Exception('EVP_DecryptFinal failed') ptl += outl + self.blocks += num_aes_blocks(inl) return out[:ptl] finally: free(out) diff --git a/borg/testsuite/crypto.py b/borg/testsuite/crypto.py index 2d74493d6..c68101942 100644 --- a/borg/testsuite/crypto.py +++ b/borg/testsuite/crypto.py @@ -1,6 +1,7 @@ from binascii import hexlify from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes +from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16 from . import BaseTestCase @@ -13,6 +14,27 @@ class CryptoTestCase(BaseTestCase): self.assert_equal(bytes_to_long(b'\0\0\0\0\0\0\0\1'), 1) self.assert_equal(long_to_bytes(1), b'\0\0\0\0\0\0\0\1') + def test_bytes16_to_int(self): + self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1'), 1) + self.assert_equal(int_to_bytes16(1), b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1') + self.assert_equal(bytes16_to_int(b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0'), 2 ** 64) + self.assert_equal(int_to_bytes16(2 ** 64), b'\0\0\0\0\0\0\0\1\0\0\0\0\0\0\0\0') + + def test_increment_iv(self): + iv0 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0' + iv1 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1' + iv2 = b'\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\2' + self.assert_equal(increment_iv(iv0, 0), iv0) + self.assert_equal(increment_iv(iv0, 1), iv1) + self.assert_equal(increment_iv(iv0, 2), iv2) + iva = b'\0\0\0\0\0\0\0\0\xff\xff\xff\xff\xff\xff\xff\xff' + ivb = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x00' + ivc = b'\0\0\0\0\0\0\0\1\x00\x00\x00\x00\x00\x00\x00\x01' + self.assert_equal(increment_iv(iva, 0), iva) + self.assert_equal(increment_iv(iva, 1), ivb) + self.assert_equal(increment_iv(iva, 2), ivc) + self.assert_equal(increment_iv(iv0, 2**64), ivb) + def test_aes(self): key = b'X' * 32 data = b'foo' * 10