mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #3319 from ThomasWaldmann/crc32-unaligned 

crc32: deal with unaligned buffer, tests, fixes #3317
This commit is contained in:
TW 2017-11-14 18:40:53 +01:00 committed by GitHub
parent b3c11dee6c f9cd6f7512
commit 72232a9bd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/borg/algorithms/crc32_slice_by_8.c
View File

@ -330,12 +330,28 @@ const uint32_t Crc32Lookup[8][256] =
uint32_t crc32_slice_by_8(const void* data, size_t length, uint32_t previousCrc32) uint32_t crc32_slice_by_8(const void* data, size_t length, uint32_t previousCrc32)
{ {
uint32_t crc = ~previousCrc32; // same as previousCrc32 ^ 0xFFFFFFFF uint32_t crc = ~previousCrc32; // same as previousCrc32 ^ 0xFFFFFFFF
const uint32_t* current = (const uint32_t*) data;
const uint32_t* current;
const uint8_t* currentChar;
// enabling optimization (at least -O2) automatically unrolls the inner for-loop // enabling optimization (at least -O2) automatically unrolls the inner for-loop
const size_t Unroll = 4; const size_t Unroll = 4;
const size_t BytesAtOnce = 8 * Unroll; const size_t BytesAtOnce = 8 * Unroll;
const uint8_t* currentChar;
currentChar = (const uint8_t*) data;
// wanted: 32 bit / 4 Byte alignment, compute leading, unaligned bytes length
uintptr_t unaligned_length = (4 - (((uintptr_t) currentChar) & 3)) & 3;
// process unaligned bytes, if any (standard algorithm)
while ((length != 0) && (unaligned_length != 0))
{
crc = (crc >> 8) ^ Crc32Lookup[0][(crc & 0xFF) ^ *currentChar++];
length--;
unaligned_length--;
}
// pointer points to 32bit aligned address now
current = (const uint32_t*) currentChar;
// process 4x eight bytes at once (Slicing-by-8) // process 4x eight bytes at once (Slicing-by-8)
while (length >= BytesAtOnce) while (length >= BytesAtOnce)

17
src/borg/testsuite/checksums.py
View File

@ -14,13 +14,18 @@ if checksums.have_clmul:
@pytest.mark.parametrize('implementation', crc32_implementations) @pytest.mark.parametrize('implementation', crc32_implementations)
def test_crc32(implementation): def test_crc32(implementation):
# This includes many critical values, like zero length, 3/4/5, 6/7/8 and so on which are near and on # This includes many critical values, like misc. length and misc. aligned start addresses.
# alignment boundaries. This is of course just a sanity check ie. "did it compile all right?". data = os.urandom(300)
data = os.urandom(256) mv = memoryview(data)
initial_crc = 0x12345678 initial_crc = 0x12345678
for i in range(0, 256): for start in range(0, 4): # 4B / int32 alignment, head processing
d = data[:i] for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc) 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
127, 128, 129, 130, 131, 132, 133, 134, 135,
255, 256, 257, ]:
d = mv[start:start+length]
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc)
def test_xxh64(): def test_xxh64():