mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-03-11 06:33:39 +00:00
Code Issues Releases Wiki Activity

Merge pull request #7598 from ThomasWaldmann/chunker-params-master

Browse source 
tests: check buzhash chunksize distribution, see #7586
This commit is contained in:
TW 2023-05-23 00:48:59 +02:00 • committed by GitHub
commit 5e4202e0d2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 27 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
src/borg/testsuite/chunker_pytest.py
View file

@ -5,7 +5,7 @@ import tempfile
import pytest
from .chunker import cf
from ..chunker import ChunkerFixed, sparsemap, has_seek_hole, ChunkerFailing
from ..chunker import Chunker, ChunkerFixed, sparsemap, has_seek_hole, ChunkerFailing
from ..constants import * # NOQA
BS = 4096 # fs block size
@ -151,3 +151,29 @@ def test_chunker_failing():
assert c1.data == data[:SIZE]
assert c2.data == data[SIZE : 2 * SIZE]
assert c3.data == data[2 * SIZE :]
def test_buzhash_chunksize_distribution():
data = os.urandom(1048576)
min_exp, max_exp, mask = 10, 16, 14 # chunk size target 16kiB, clip at 1kiB and 64kiB
chunker = Chunker(0, min_exp, max_exp, mask, 4095)
f = BytesIO(data)
chunks = cf(chunker.chunkify(f))
chunk_sizes = [len(chunk) for chunk in chunks]
chunks_count = len(chunks)
min_chunksize_observed = min(chunk_sizes)
max_chunksize_observed = max(chunk_sizes)
min_count = sum((int(size == 2**min_exp) for size in chunk_sizes))
max_count = sum((int(size == 2**max_exp) for size in chunk_sizes))
print(
f"count: {chunks_count} min: {min_chunksize_observed} max: {max_chunksize_observed} "
f"min count: {min_count} max count: {max_count}"
)
# usually there will about 64 chunks
assert 32 < chunks_count < 128
# chunks always must be between min and max (clipping must work):
assert min_chunksize_observed >= 2**min_exp
assert max_chunksize_observed <= 2**max_exp
# most chunks should be cut due to buzhash triggering, not due to clipping at min/max size:
assert min_count < 10
assert max_count < 10