Merge pull request #2476 from enkore/f/pkgstruct

Create borg.algorithms and borg.crypto packages

.gitignore

@@ -4,23 +4,23 @@ build
 dist
 borg-env
 .tox
-hashindex.c
-chunker.c
-compress.c
-crypto.c
-item.c
-src/borg/crc32.c
+src/borg/compress.c
+src/borg/crypto/low_level.c
+src/borg/hashindex.c
+src/borg/item.c
+src/borg/algorithms/chunker.c
+src/borg/algorithms/crc32.c
 src/borg/platform/darwin.c
 src/borg/platform/freebsd.c
 src/borg/platform/linux.c
 src/borg/platform/posix.c
+src/borg/_version.py
 *.egg-info
 *.pyc
 *.pyo
 *.so
 .idea/
 .cache/
-src/borg/_version.py
 borg.build/
 borg.dist/
 borg.exe

setup.py

@@ -50,11 +50,11 @@
 from setuptools.command.sdist import sdist
 
 compress_source = 'src/borg/compress.pyx'
-crypto_source = 'src/borg/crypto.pyx'
-chunker_source = 'src/borg/chunker.pyx'
+crypto_ll_source = 'src/borg/crypto/low_level.pyx'
+chunker_source = 'src/borg/algorithms/chunker.pyx'
 hashindex_source = 'src/borg/hashindex.pyx'
 item_source = 'src/borg/item.pyx'
-crc32_source = 'src/borg/crc32.pyx'
+crc32_source = 'src/borg/algorithms/crc32.pyx'
 platform_posix_source = 'src/borg/platform/posix.pyx'
 platform_linux_source = 'src/borg/platform/linux.pyx'
 platform_darwin_source = 'src/borg/platform/darwin.pyx'
@@ -62,7 +62,7 @@
 
 cython_sources = [
     compress_source,
-    crypto_source,
+    crypto_ll_source,
     chunker_source,
     hashindex_source,
     item_source,
@@ -87,12 +87,12 @@ def __init__(self, *args, **kwargs):
         def make_distribution(self):
             self.filelist.extend([
                 'src/borg/compress.c',
-                'src/borg/crypto.c',
-                'src/borg/chunker.c', 'src/borg/_chunker.c',
+                'src/borg/crypto/low_level.c',
+                'src/borg/algorithms/chunker.c', 'src/borg/algorithms/buzhash.c',
                 'src/borg/hashindex.c', 'src/borg/_hashindex.c',
                 'src/borg/item.c',
-                'src/borg/crc32.c',
-                'src/borg/_crc32/crc32.c', 'src/borg/_crc32/clmul.c', 'src/borg/_crc32/slice_by_8.c',
+                'src/borg/algorithms/crc32.c',
+                'src/borg/algorithms/crc32_dispatch.c', 'src/borg/algorithms/crc32_clmul.c', 'src/borg/algorithms/crc32_slice_by_8.c',
                 'src/borg/platform/posix.c',
                 'src/borg/platform/linux.c',
                 'src/borg/platform/freebsd.c',
@@ -106,7 +106,7 @@ def __init__(self, *args, **kwargs):
             raise Exception('Cython is required to run sdist')
 
     compress_source = compress_source.replace('.pyx', '.c')
-    crypto_source = crypto_source.replace('.pyx', '.c')
+    crypto_ll_source = crypto_ll_source.replace('.pyx', '.c')
     chunker_source = chunker_source.replace('.pyx', '.c')
     hashindex_source = hashindex_source.replace('.pyx', '.c')
     item_source = item_source.replace('.pyx', '.c')
@@ -117,7 +117,7 @@ def __init__(self, *args, **kwargs):
     platform_darwin_source = platform_darwin_source.replace('.pyx', '.c')
     from distutils.command.build_ext import build_ext
     if not on_rtd and not all(os.path.exists(path) for path in [
-        compress_source, crypto_source, chunker_source, hashindex_source, item_source, crc32_source,
+        compress_source, crypto_ll_source, chunker_source, hashindex_source, item_source, crc32_source,
         platform_posix_source, platform_linux_source, platform_freebsd_source, platform_darwin_source]):
         raise ImportError('The GIT version of Borg needs Cython. Install Cython or use a released version.')
 
@@ -578,11 +578,12 @@ def is_positional_group(group):
 if not on_rtd:
     ext_modules += [
     Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
-    Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
-    Extension('borg.chunker', [chunker_source]),
+    Extension('borg.crypto', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
+    Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
     Extension('borg.hashindex', [hashindex_source]),
     Extension('borg.item', [item_source]),
-    Extension('borg.crc32', [crc32_source]),
+    Extension('borg.algorithms.chunker', [chunker_source]),
+    Extension('borg.algorithms.crc32', [crc32_source]),
 ]
     if not sys.platform.startswith(('win32', )):
         ext_modules.append(Extension('borg.platform.posix', [platform_posix_source]))

src/borg/algorithms/chunker.pyx

@@ -4,7 +4,7 @@ API_VERSION = '1.1_01'
 
 from libc.stdlib cimport free
 
-cdef extern from "_chunker.c":
+cdef extern from "buzhash.c":
     ctypedef int uint32_t
     ctypedef struct _Chunker "Chunker":
         pass

src/borg/algorithms/crc32.pyx

@@ -3,7 +3,7 @@ from libc.stdint cimport uint32_t
 from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 
 
-cdef extern from "_crc32/crc32.c":
+cdef extern from "crc32_dispatch.c":
     uint32_t _crc32_slice_by_8 "crc32_slice_by_8"(const void* data, size_t length, uint32_t initial_crc)
     uint32_t _crc32_clmul "crc32_clmul"(const void* data, size_t length, uint32_t initial_crc)
 

src/borg/algorithms/crc32_dispatch.c

@@ -1,6 +1,6 @@
 
 /* always compile slice by 8 as a runtime fallback */
-#include "slice_by_8.c"
+#include "crc32_slice_by_8.c"
 
 #ifdef __GNUC__
 /*
@@ -69,7 +69,7 @@
 #endif /* ifdef __GNUC__ */
 
 #ifdef FOLDING_CRC
-#include "clmul.c"
+#include "crc32_clmul.c"
 #else
 
 static uint32_t

src/borg/archive.py

@@ -16,11 +16,13 @@
 import msgpack
 
 from .logger import create_logger
+
 logger = create_logger()
 
 from . import xattr
+from .algorithms.chunker import Chunker
 from .cache import ChunkListEntry
-from .chunker import Chunker
+from .crypto.key import key_factory
 from .compress import Compressor, CompressionSpec
 from .constants import *  # NOQA
 from .hashindex import ChunkIndex, ChunkIndexEntry
@@ -38,7 +40,6 @@
 from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem
-from .key import key_factory
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth
 from .remote import cache_if_remote
 from .repository import Repository, LIST_SCAN_LIMIT

src/borg/archiver.py

@@ -24,6 +24,7 @@
 from itertools import zip_longest
 
 from .logger import create_logger, setup_logging
+
 logger = create_logger()
 
 import msgpack
@@ -31,21 +32,23 @@
 import borg
 from . import __version__
 from . import helpers
+from .algorithms.crc32 import crc32
 from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics, is_special
 from .archive import BackupOSError, backup_io
 from .cache import Cache
 from .constants import *  # NOQA
 from .compress import CompressionSpec
-from .crc32 import crc32
+from .crypto.key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
+from .crypto.keymanager import KeyManager
 from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from .helpers import Error, NoManifestError, set_ec
 from .helpers import location_validator, archivename_validator, ChunkerParams
 from .helpers import PrefixSpec, SortBySpec, HUMAN_SORT_KEYS
 from .helpers import BaseFormatter, ItemFormatter, ArchiveFormatter
-from .helpers import format_time, format_timedelta, format_file_size, format_archive
+from .helpers import format_timedelta, format_file_size, format_archive
 from .helpers import safe_encode, remove_surrogates, bin_to_hex, prepare_dump_dict
 from .helpers import prune_within, prune_split
-from .helpers import to_localtime, timestamp
+from .helpers import timestamp
 from .helpers import get_cache_dir
 from .helpers import Manifest
 from .helpers import hardlinkable
@@ -61,8 +64,6 @@
 from .patterns import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
 from .patterns import PatternMatcher
 from .item import Item
-from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
-from .keymanager import KeyManager
 from .platform import get_flags, umount, get_process_id, SyncFile
 from .remote import RepositoryServer, RemoteRepository, cache_if_remote
 from .repository import Repository, LIST_SCAN_LIMIT

src/borg/cache.py

@@ -1,13 +1,14 @@
 import configparser
 import os
-import stat
 import shutil
+import stat
 from binascii import unhexlify
 from collections import namedtuple
 
 import msgpack
 
 from .logger import create_logger
+
 logger = create_logger()
 
 from .constants import CACHE_README
@@ -21,8 +22,8 @@
 from .helpers import yes, hostname_is_unique
 from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
-from .item import Item, ArchiveItem, ChunkListEntry
-from .key import PlaintextKey
+from .item import ArchiveItem, ChunkListEntry
+from .crypto.key import PlaintextKey
 from .locking import Lock
 from .platform import SaveFile
 from .remote import cache_if_remote

src/borg/crypto/key.py

@@ -3,27 +3,27 @@
 import os
 import sys
 import textwrap
-from binascii import a2b_base64, b2a_base64, hexlify, unhexlify
+from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, sha512, pbkdf2_hmac
 from hmac import HMAC, compare_digest
 
 import msgpack
 
-from .logger import create_logger
+from borg.logger import create_logger
+
 logger = create_logger()
 
-from .constants import *  # NOQA
-from .compress import Compressor
-from .crypto import AES, bytes_to_long, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
-from .helpers import StableDict
-from .helpers import Error, IntegrityError
-from .helpers import yes
-from .helpers import get_keys_dir, get_security_dir
-from .helpers import bin_to_hex
-from .item import Key, EncryptedKey
-from .platform import SaveFile
+from ..constants import *  # NOQA
+from ..compress import Compressor
+from ..helpers import StableDict
+from ..helpers import Error, IntegrityError
+from ..helpers import yes
+from ..helpers import get_keys_dir, get_security_dir
+from ..helpers import bin_to_hex
+from ..item import Key, EncryptedKey
+from ..platform import SaveFile
 from .nonces import NonceManager
-
+from .low_level import AES, bytes_to_long, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
 
 PREFIX = b'\0' * 8
 

src/borg/crypto/keymanager.py

@@ -1,12 +1,13 @@
-from binascii import unhexlify, a2b_base64, b2a_base64
 import binascii
-import textwrap
-from hashlib import sha256
 import pkgutil
+import textwrap
+from binascii import unhexlify, a2b_base64, b2a_base64
+from hashlib import sha256
+
+from ..helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
+from ..repository import Repository
 
 from .key import KeyfileKey, KeyfileNotFoundError, KeyBlobStorage, identify_key
-from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
-from .repository import Repository
 
 
 class UnencryptedRepo(Error):

src/borg/crypto/low_level.pyx

@@ -10,7 +10,7 @@ from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release
 API_VERSION = '1.1_01'
 
 
-cdef extern from "blake2-libselect.h":
+cdef extern from "../algorithms/blake2-libselect.h":
     ctypedef struct blake2b_state:
         pass
 

src/borg/crypto/nonces.py

@@ -2,12 +2,12 @@
 import sys
 from binascii import unhexlify
 
-from .crypto import bytes_to_long, long_to_bytes
-from .helpers import get_security_dir
-from .helpers import bin_to_hex
-from .platform import SaveFile
-from .remote import InvalidRPCMethod
+from ..helpers import get_security_dir
+from ..helpers import bin_to_hex
+from ..platform import SaveFile
+from ..remote import InvalidRPCMethod
 
+from .low_level import bytes_to_long, long_to_bytes
 
 MAX_REPRESENTABLE_NONCE = 2**64 - 1
 NONCE_SPACE_RESERVATION = 2**28  # This in units of AES blocksize (16 bytes)

src/borg/helpers.py

@@ -1,11 +1,11 @@
 import argparse
-import contextlib
 import collections
+import contextlib
 import grp
 import hashlib
-import logging
 import io
 import json
+import logging
 import os
 import os.path
 import platform
@@ -25,20 +25,21 @@
 from functools import partial, lru_cache
 from itertools import islice
 from operator import attrgetter
-from string import Formatter
 from shutil import get_terminal_size
+from string import Formatter
 
 import msgpack
 import msgpack.fallback
 
 from .logger import create_logger
+
 logger = create_logger()
 
+import borg.crypto.low_level
 from . import __version__ as borg_version
 from . import __version_tuple__ as borg_version_tuple
-from . import chunker
-from . import crypto
 from . import hashindex
+from .algorithms import chunker
 from .constants import *  # NOQA
 
 
@@ -119,7 +120,7 @@ def check_extension_modules():
         raise ExtensionModuleError
     if compress.API_VERSION != '1.1_03':
         raise ExtensionModuleError
-    if crypto.API_VERSION != '1.1_01':
+    if borg.crypto.low_level.API_VERSION != '1.1_01':
         raise ExtensionModuleError
     if platform.API_VERSION != platform.OS_API_VERSION != '1.1_01':
         raise ExtensionModuleError
@@ -232,7 +233,7 @@ def last_timestamp(self):
     @classmethod
     def load(cls, repository, key=None, force_tam_not_required=False):
         from .item import ManifestItem
-        from .key import key_factory, tam_required_file, tam_required
+        from .crypto.key import key_factory, tam_required_file, tam_required
         from .repository import Repository
         try:
             cdata = repository.get(cls.MANIFEST_ID)

src/borg/repository.py

@@ -23,7 +23,7 @@
 from .logger import create_logger
 from .lrucache import LRUCache
 from .platform import SaveFile, SyncFile, sync_dir, safe_fadvise
-from .crc32 import crc32
+from .algorithms.crc32 import crc32
 
 logger = create_logger(__name__)
 

src/borg/testsuite/archive.py

@@ -1,18 +1,17 @@
-import os
 from collections import OrderedDict
 from datetime import datetime, timezone
 from io import StringIO
 from unittest.mock import Mock
 
-import pytest
 import msgpack
+import pytest
 
+from . import BaseTestCase
+from ..crypto.key import PlaintextKey
 from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
 from ..archive import BackupOSError, backup_io, backup_io_iter
-from ..item import Item, ArchiveItem
-from ..key import PlaintextKey
 from ..helpers import Manifest
-from . import BaseTestCase
+from ..item import Item, ArchiveItem
 
 
 @pytest.fixture()

src/borg/testsuite/archiver.py

@@ -1,38 +1,40 @@
-from binascii import unhexlify, b2a_base64
-from configparser import ConfigParser
 import errno
-import os
-import inspect
 import json
-from datetime import datetime
-from datetime import timedelta
-from io import StringIO
 import logging
+import os
 import random
+import shutil
 import socket
 import stat
 import subprocess
 import sys
-import shutil
 import tempfile
 import time
 import unittest
-from unittest.mock import patch
+from binascii import unhexlify, b2a_base64
+from configparser import ConfigParser
+from datetime import datetime
+from datetime import timedelta
 from hashlib import sha256
+from io import StringIO
+from unittest.mock import patch
 
 import msgpack
 import pytest
+
 try:
     import llfuse
 except ImportError:
     pass
 
 from .. import xattr, helpers, platform
-from ..archive import Archive, ChunkBuffer, ArchiveRecreater, flags_noatime, flags_normal
+from ..archive import Archive, ChunkBuffer, flags_noatime, flags_normal
 from ..archiver import Archiver
 from ..cache import Cache
 from ..constants import *  # NOQA
-from ..crypto import bytes_to_long, num_aes_blocks
+from ..crypto.low_level import bytes_to_long, num_aes_blocks
+from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
+from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..helpers import Location, get_security_dir
 from ..helpers import Manifest
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
@@ -40,8 +42,6 @@
 from ..helpers import MAX_S
 from ..patterns import IECommand, PatternMatcher, parse_pattern
 from ..item import Item
-from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
-from ..keymanager import RepoIdMismatch, NotABorgKeyFile
 from ..remote import RemoteRepository, PathNotAllowed
 from ..repository import Repository
 from . import has_lchflags, has_llfuse

src/borg/testsuite/chunker.py

@@ -1,6 +1,6 @@
 from io import BytesIO
 
-from ..chunker import Chunker, buzhash, buzhash_update
+from ..algorithms.chunker import Chunker, buzhash, buzhash_update
 from ..constants import *  # NOQA
 from . import BaseTestCase
 

src/borg/testsuite/crc32.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-from .. import crc32
+from ..algorithms import crc32
 
 crc32_implementations = [crc32.crc32_slice_by_8]
 if crc32.have_clmul:

src/borg/testsuite/crypto.py

@@ -1,8 +1,8 @@
 from binascii import hexlify, unhexlify
 
-from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256
-from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16
-from ..crypto import hkdf_hmac_sha512
+from ..crypto.low_level import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256
+from ..crypto.low_level import increment_iv, bytes16_to_int, int_to_bytes16
+from ..crypto.low_level import hkdf_hmac_sha512
 from . import BaseTestCase
 
 # Note: these tests are part of the self test, do not use or import py.test functionality here.

src/borg/testsuite/key.py

@@ -1,20 +1,21 @@
 import getpass
+import os.path
 import re
 import tempfile
-import os.path
 from binascii import hexlify, unhexlify
 
-import pytest
 import msgpack
+import pytest
 
-from ..crypto import bytes_to_long, num_aes_blocks
+from ..crypto.key import Passphrase, PasswordRetriesExceeded, bin_to_hex
+from ..crypto.key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, \
+    AuthenticatedKey
+from ..crypto.key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError
+from ..crypto.low_level import bytes_to_long, num_aes_blocks
+from ..helpers import IntegrityError
 from ..helpers import Location
 from ..helpers import StableDict
-from ..helpers import IntegrityError
 from ..helpers import get_security_dir
-from ..key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, AuthenticatedKey
-from ..key import Passphrase, PasswordRetriesExceeded, bin_to_hex
-from ..key import TAMRequiredError, TAMInvalid, TAMUnsupportedSuiteError, UnsupportedManifestError
 
 
 class TestKey:

src/borg/testsuite/nonces.py

@@ -2,13 +2,12 @@
 
 import pytest
 
+from ..crypto import nonces
+from ..crypto.nonces import NonceManager
+from ..crypto.key import bin_to_hex
 from ..helpers import get_security_dir
-from ..key import bin_to_hex
-from ..nonces import NonceManager
 from ..remote import InvalidRPCMethod
 
-from .. import nonces  # for monkey patching NONCE_SPACE_RESERVATION
-
 
 class TestNonceManager:
 

src/borg/testsuite/upgrader.py

@@ -10,9 +10,9 @@
     attic = None
 
 from ..constants import *  # NOQA
+from ..crypto.key import KeyfileKey
 from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
 from ..helpers import get_keys_dir
-from ..key import KeyfileKey
 from ..repository import Repository
 from . import are_hardlinks_supported
 

src/borg/upgrader.py

@@ -3,13 +3,13 @@
 import shutil
 import time
 
+from .crypto.key import KeyfileKey, KeyfileNotFoundError
 from .constants import REPOSITORY_README
-from .helpers import get_home_dir, get_keys_dir, get_cache_dir
 from .helpers import ProgressIndicatorPercent
-from .key import KeyfileKey, KeyfileNotFoundError
+from .helpers import get_home_dir, get_keys_dir, get_cache_dir
 from .locking import Lock
-from .repository import Repository, MAGIC
 from .logger import create_logger
+from .repository import Repository, MAGIC
 
 logger = create_logger(__name__)