mirror of https://github.com/borgbackup/borg.git
stop using libdeflate
borg2's new repo format does not need computing crc32 over big amounts of (content) data any more (we now use xxh64 for that). thus, having a quick crc32 implementation via libdeflate is not important enough any more to rectify having libdeflate as a requirement.
This commit is contained in:
Thomas Waldmann
parent
820a927b06
commit
5c8a5f111f
|
|
@ -67,7 +67,6 @@ jobs:
|
|||
env:
|
||||
# Configure pkg-config to use OpenSSL from Homebrew
|
||||
PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig
|
||||
BORG_LIBDEFLATE_PREFIX: /usr # on ubuntu 20.04 pkgconfig does not find libdeflate
|
||||
TOXENV: ${{ matrix.toxenv }}
|
||||
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
|
@ -96,7 +95,7 @@ jobs:
|
|||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y pkg-config build-essential
|
||||
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev
|
||||
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
|
||||
sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module
|
||||
sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module
|
||||
|
||||
|
|
@ -106,7 +105,6 @@ jobs:
|
|||
brew install pkg-config || brew upgrade pkg-config
|
||||
brew install zstd || brew upgrade zstd
|
||||
brew install lz4 || brew upgrade lz4
|
||||
brew install libdeflate || brew upgrade libdeflate
|
||||
brew install xxhash || brew upgrade xxhash
|
||||
brew install openssl@1.1 || brew upgrade openssl@1.1
|
||||
brew install --cask macfuse || brew upgrade --cask macfuse # Required for Python llfuse module
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ jobs:
|
|||
# Learn more about CodeQL language support at https://git.io/codeql-language-support
|
||||
|
||||
env:
|
||||
BORG_LIBDEFLATE_PREFIX: /usr # on ubuntu 20.04 pkgconfig does not find libdeflate
|
||||
# nothing special right now
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
|
|
@ -52,7 +52,7 @@ jobs:
|
|||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y pkg-config build-essential
|
||||
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev
|
||||
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v1
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ def packages_debianoid(user)
|
|||
apt-get -y -qq update
|
||||
apt-get -y -qq dist-upgrade
|
||||
# for building borgbackup and dependencies:
|
||||
apt install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev pkg-config
|
||||
apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev pkg-config
|
||||
apt install -y libfuse-dev fuse || true
|
||||
apt install -y libfuse3-dev fuse3 || true
|
||||
apt install -y locales || true
|
||||
|
|
@ -29,7 +29,6 @@ def packages_debianoid(user)
|
|||
apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev
|
||||
# older debian / ubuntu have no .pc file for these, so we need to point at the lib/header location:
|
||||
echo 'export BORG_LIBXXHASH_PREFIX=/usr' >> ~vagrant/.bash_profile
|
||||
echo 'export BORG_LIBDEFLATE_PREFIX=/usr' >> ~vagrant/.bash_profile
|
||||
EOF
|
||||
end
|
||||
|
||||
|
|
@ -40,7 +39,7 @@ def packages_freebsd
|
|||
# install all the (security and other) updates, base system
|
||||
freebsd-update --not-running-from-cron fetch install
|
||||
# for building borgbackup and dependencies:
|
||||
pkg install -y xxhash libdeflate liblz4 zstd pkgconf
|
||||
pkg install -y xxhash liblz4 zstd pkgconf
|
||||
pkg install -y fusefs-libs || true
|
||||
pkg install -y fusefs-libs3 || true
|
||||
pkg install -y git bash # fakeroot causes lots of troubles on freebsd
|
||||
|
|
@ -74,7 +73,6 @@ def packages_openbsd
|
|||
pkg_add bash
|
||||
chsh -s bash vagrant
|
||||
pkg_add xxhash
|
||||
pkg_add libdeflate
|
||||
pkg_add lz4
|
||||
pkg_add zstd
|
||||
pkg_add git # no fakeroot
|
||||
|
|
@ -119,7 +117,7 @@ def packages_darwin
|
|||
sudo softwareupdate --install --all
|
||||
which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
|
||||
brew update > /dev/null
|
||||
brew install pkg-config readline openssl@1.1 xxhash libdeflate zstd lz4 xz
|
||||
brew install pkg-config readline openssl@1.1 xxhash zstd lz4 xz
|
||||
brew install --cask macfuse
|
||||
# brew upgrade # upgrade everything (takes rather long)
|
||||
echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@1.1/lib/pkgconfig' >> ~vagrant/.bash_profile
|
||||
|
|
|
|||
|
|
@ -167,7 +167,6 @@ following dependencies first:
|
|||
- liblz4_ >= 1.7.0 (r129)
|
||||
- libzstd_ >= 1.3.0
|
||||
- libxxhash >= 0.8.1 (0.8.0 might work also)
|
||||
- libdeflate >= 1.5
|
||||
* pkg-config (cli tool) and pkgconfig python package (borg uses these to
|
||||
discover header and library location - if it can't import pkgconfig and
|
||||
is not pointed to header/library locations via env vars [see setup.py],
|
||||
|
|
@ -202,7 +201,7 @@ Install the dependencies with development headers::
|
|||
sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \
|
||||
libacl1-dev libacl1 \
|
||||
libssl-dev \
|
||||
liblz4-dev libzstd-dev libxxhash-dev libdeflate-dev \
|
||||
liblz4-dev libzstd-dev libxxhash-dev \
|
||||
build-essential \
|
||||
pkg-config python3-pkgconfig
|
||||
sudo apt-get install libfuse-dev fuse # needed for llfuse
|
||||
|
|
@ -220,7 +219,7 @@ Install the dependencies with development headers::
|
|||
sudo dnf install python3 python3-devel python3-pip python3-virtualenv \
|
||||
libacl-devel libacl \
|
||||
openssl-devel \
|
||||
lz4-devel libzstd-devel xxhash-devel libdeflate-devel \
|
||||
lz4-devel libzstd-devel xxhash-devel \
|
||||
pkgconf python3-pkgconfig
|
||||
sudo dnf install gcc gcc-c++ redhat-rpm-config
|
||||
sudo dnf install fuse-devel fuse # needed for llfuse
|
||||
|
|
@ -236,8 +235,7 @@ Install the dependencies automatically using zypper::
|
|||
Alternatively, you can enumerate all build dependencies in the command line::
|
||||
|
||||
sudo zypper install python3 python3-devel \
|
||||
libacl-devel openssl-devel \
|
||||
libxxhash-devel libdeflate-devel \
|
||||
libacl-devel openssl-devel libxxhash-devel \
|
||||
python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \
|
||||
python3-pytest python3-setuptools python3-setuptools_scm \
|
||||
python3-sphinx_rtd_theme gcc gcc-c++
|
||||
|
|
@ -249,7 +247,7 @@ macOS
|
|||
When installing via Homebrew_, dependencies are installed automatically. To install
|
||||
dependencies manually::
|
||||
|
||||
brew install python3 openssl zstd lz4 xxhash libdeflate
|
||||
brew install python3 openssl zstd lz4 xxhash
|
||||
brew install pkg-config
|
||||
pip3 install virtualenv pkgconfig
|
||||
|
||||
|
|
@ -286,7 +284,7 @@ and commands to make FUSE work for using the mount command.
|
|||
|
||||
pkg install -y python3 pkgconf
|
||||
pkg install openssl
|
||||
pkg install liblz4 zstd xxhash libdeflate
|
||||
pkg install liblz4 zstd xxhash
|
||||
pkg install fusefs-libs # needed for llfuse
|
||||
pkg install -y git
|
||||
python3 -m ensurepip # to install pip for Python3
|
||||
|
|
@ -316,7 +314,7 @@ Use the Cygwin installer to install the dependencies::
|
|||
|
||||
python39 python39-devel python39-pkgconfig
|
||||
python39-setuptools python39-pip python39-wheel python39-virtualenv
|
||||
libssl-devel libxxhash-devel libdeflate-devel liblz4-devel libzstd-devel
|
||||
libssl-devel libxxhash-devel liblz4-devel libzstd-devel
|
||||
binutils gcc-g++ git make openssh
|
||||
|
||||
|
||||
|
|
|
|||
1
setup.py
1
setup.py
|
|
@ -186,7 +186,6 @@ if not on_rtd:
|
|||
checksums_ext_kwargs = members_appended(
|
||||
dict(sources=[checksums_source]),
|
||||
lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'),
|
||||
lib_ext_kwargs(pc, 'BORG_LIBDEFLATE_PREFIX', 'deflate', 'libdeflate', '>= 1.5'),
|
||||
dict(extra_compile_args=cflags),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -715,21 +715,13 @@ class Archiver:
|
|||
]:
|
||||
print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
|
||||
|
||||
import zlib
|
||||
from .checksums import crc32, deflate_crc32, xxh64
|
||||
from .checksums import crc32, xxh64
|
||||
print("Non-cryptographic checksums / hashes ===========================")
|
||||
size = "1GB"
|
||||
tests = [
|
||||
("xxh64", lambda: xxh64(random_10M)),
|
||||
("crc32 (zlib)", lambda: crc32(random_10M)),
|
||||
]
|
||||
if crc32 is zlib.crc32:
|
||||
tests.insert(0, ("crc32 (zlib, used)", lambda: crc32(random_10M)))
|
||||
tests.insert(1, ("crc32 (libdeflate)", lambda: deflate_crc32(random_10M)))
|
||||
elif crc32 is deflate_crc32:
|
||||
tests.insert(0, ("crc32 (libdeflate, used)", lambda: crc32(random_10M)))
|
||||
tests.insert(1, ("crc32 (zlib)", lambda: zlib.crc32(random_10M)))
|
||||
else:
|
||||
raise Error("crc32 benchmarking code missing")
|
||||
for spec, func in tests:
|
||||
print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s")
|
||||
|
||||
|
|
|
|||
|
|
@ -33,31 +33,15 @@ cdef extern from "xxhash.h":
|
|||
XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src)
|
||||
|
||||
|
||||
cdef extern from "libdeflate.h":
|
||||
uint32_t libdeflate_crc32(uint32_t crc, const void *buffer, size_t len)
|
||||
|
||||
|
||||
cdef Py_buffer ro_buffer(object data) except *:
|
||||
cdef Py_buffer view
|
||||
PyObject_GetBuffer(data, &view, PyBUF_SIMPLE)
|
||||
return view
|
||||
|
||||
|
||||
def deflate_crc32(data, value=0):
|
||||
cdef Py_buffer data_buf = ro_buffer(data)
|
||||
cdef uint32_t val = value
|
||||
try:
|
||||
return libdeflate_crc32(val, data_buf.buf, data_buf.len)
|
||||
finally:
|
||||
PyBuffer_Release(&data_buf)
|
||||
|
||||
|
||||
if is_darwin:
|
||||
# macOS (darwin) has a highly optimized zlib.crc32 (Intel as well as Apple Silicon M1)
|
||||
crc32 = zlib.crc32
|
||||
else:
|
||||
# on Linux x64 (and maybe others), libdeflate_crc32 is faster than zlib.crc32
|
||||
crc32 = deflate_crc32
|
||||
# borg 2.0's new repos do not compute crc32 over big amounts of data,
|
||||
# so speed does not matter much any more and we can just use zlib.crc32.
|
||||
crc32 = zlib.crc32
|
||||
|
||||
|
||||
def xxh64(data, seed=0):
|
||||
|
|
|
|||
|
|
@ -1,30 +1,8 @@
|
|||
import os
|
||||
import zlib
|
||||
from binascii import unhexlify
|
||||
|
||||
import pytest
|
||||
|
||||
from .. import checksums
|
||||
from ..helpers import bin_to_hex
|
||||
|
||||
crc32_implementations = [checksums.deflate_crc32]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('implementation', crc32_implementations)
|
||||
def test_crc32(implementation):
|
||||
# This includes many critical values, like misc. length and misc. aligned start addresses.
|
||||
data = os.urandom(300)
|
||||
mv = memoryview(data)
|
||||
initial_crc = 0x12345678
|
||||
for start in range(0, 4): # 4B / int32 alignment, head processing
|
||||
for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
|
||||
31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
|
||||
63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
|
||||
127, 128, 129, 130, 131, 132, 133, 134, 135,
|
||||
255, 256, 257, ]:
|
||||
d = mv[start:start+length]
|
||||
assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc)
|
||||
|
||||
|
||||
def test_xxh64():
|
||||
assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'
|
||||
|
|
|
|||
Loading…
Reference in New Issue