add mypy checking

also added some .pyi files needed to check the cython code (taken from #5703 and updated).

fixed "syntax error" in key.py.

all mypy complaints not fixed yet.

.github/workflows/ci.yml

@@ -45,12 +45,15 @@ jobs:
         pip install flake8
         flake8 src scripts conftest.py
 
-  pytest:
+  tox:
 
     needs: lint
     strategy:
       matrix:
         include:
+            - os: ubuntu-20.04
+              python-version: '3.9'
+              toxenv: mypy
             - os: ubuntu-20.04
               python-version: '3.9'
               toxenv: py39-fuse2
@@ -117,7 +120,7 @@ jobs:
       run: |
         # pip install -e .
         python setup.py -v develop
-    - name: run pytest via tox
+    - name: run tox env
       run: |
         # do not use fakeroot, but run as root. avoids the dreaded EISDIR sporadic failures. see #2482.
         #sudo -E bash -c "tox -e py"

setup.cfg

@@ -177,3 +177,23 @@ per_file_ignores =
 max_line_length = 120
 exclude = build,dist,.git,.idea,.cache,.tox
 
+[mypy]
+python_version = 3.9
+strict_optional = False
+local_partial_types = True
+show_error_codes = True
+files = src/borg/**/*.py
+
+[mypy-msgpack.*]
+ignore_missing_imports = True
+[mypy-llfuse]
+ignore_missing_imports = True
+[mypy-pyfuse3]
+ignore_missing_imports = True
+[mypy-trio]
+ignore_missing_imports = True
+
+[mypy-borg.crypto.low_level]
+ignore_missing_imports = True
+[mypy-borg.platform.*]
+ignore_missing_imports = True

src/borg/checksums.pyi

@@ -0,0 +1,8 @@
+def crc32(data: bytes, value: int = 0) -> int: ...
+def xxh64(data: bytes, seed: int = 0) -> bytes: ...
+
+class StreamingXXH64:
+    def __init__(self, seed: int = 0) -> None: ...
+    def update(self, data: bytes) -> None: ...
+    def digest(self) -> bytes: ...
+    def hexdigest(self) -> str: ...

src/borg/chunker.pyi

@@ -0,0 +1,28 @@
+from typing import NamedTuple, Tuple, List, Dict, Any, Type, Iterator, BinaryIO
+
+API_VERSION: str
+
+has_seek_hole: bool
+
+class _Chunk(NamedTuple):
+    data: bytes
+    meta: Dict[str, Any]
+
+def Chunk(data: bytes, **meta) -> Type[_Chunk]: ...
+def buzhash(data: bytes, seed: int) -> int: ...
+def buzhash_update(sum: int, remove: int, add: int, len: int, seed: int) -> int: ...
+def get_chunker(algo: str, *params, **kw) -> Any: ...
+
+fmap_entry = Tuple[int, int, bool]
+
+def sparsemap(fd: BinaryIO = None, fh: int = -1) -> List[fmap_entry]: ...
+
+class ChunkerFixed:
+    def __init__(self, block_size: int, header_size: int = 0, sparse: bool = False) -> None: ...
+    def chunkify(self, fd: BinaryIO = None, fh: int = -1, fmap: List[fmap_entry] = None) -> Iterator: ...
+
+class Chunker:
+    def __init__(
+        self, seed: int, chunk_min_exp: int, chunk_max_exp: int, hash_mask_bits: int, hash_window_size: int
+    ) -> None: ...
+    def chunkify(self, fd: BinaryIO = None, fh: int = -1) -> Iterator: ...

src/borg/compress.pyi

@@ -0,0 +1,63 @@
+from typing import Any, Type
+
+API_VERSION: str
+
+def get_compressor(name: str, **kwargs) -> Any: ...
+
+class CompressionSpec:
+    def __init__(self, spec: str) -> None: ...
+    @property
+    def compressor(self) -> Any: ...
+    inner: CompressionSpec
+
+class Compressor:
+    def __init__(self, name: Any = ..., **kwargs) -> None: ...
+    def compress(self, data: bytes) -> bytes: ...
+    def decompress(self, data: bytes) -> bytes: ...
+    @staticmethod
+    def detect(data: bytes) -> Any: ...
+
+class CompressorBase:
+    ID: bytes = ...
+    name: str = ...
+    @classmethod
+    def detect(self, data: bytes) -> bool: ...
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    def decide(self, data: bytes) -> Any: ...
+    def compress(self, data: bytes) -> bytes: ...
+    def decompress(self, data: bytes) -> bytes: ...
+
+class Auto(CompressorBase):
+    def __init__(self, compressor: Any) -> None: ...
+
+class DecidingCompressor(CompressorBase):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    def decide_compress(self, data: bytes) -> Any: ...
+
+class CNONE(CompressorBase):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+
+class ObfuscateSize(CompressorBase):
+    def __init__(self, level: int = ..., compressor: Any = ...) -> None: ...
+
+class ZLIB_legacy(CompressorBase):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    level: int
+
+class ZLIB(CompressorBase):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    level: int
+
+class LZ4(DecidingCompressor):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+
+class LZMA(DecidingCompressor):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    level: int
+
+class ZSTD(DecidingCompressor):
+    def __init__(self, level: int = ..., **kwargs) -> None: ...
+    level: int
+
+LZ4_COMPRESSOR: Type[LZ4]
+NONE_COMPRESSOR: Type[CNONE]

src/borg/crypto/key.py

@@ -153,7 +153,7 @@ class KeyBase:
     STORAGE = KeyBlobStorage.NO_STORAGE
 
     # Seed for the buzhash chunker (borg.algorithms.chunker.Chunker)
-    # type: int
+    # type is int
     chunk_seed = None
 
     # Whether this *particular instance* is encrypted from a practical point of view,

src/borg/hashindex.pyi

@@ -0,0 +1,88 @@
+from typing import NamedTuple, Tuple, Type, Union, IO, Iterator, Any
+
+API_VERSION: str
+
+PATH_OR_FILE = Union[str, IO]
+
+def hashindex_variant(fn: str) -> str: ...
+
+class IndexBase:
+    value_size: int
+    MAX_VALUE: int
+    MAX_LOAD_FACTOR: int
+    def __init__(
+        self, capacity: int = ..., path: PATH_OR_FILE = ..., permit_compact: bool = ..., usable: Union[int, float] = ...
+    ): ...
+    @classmethod
+    def read(cls, path: PATH_OR_FILE, permit_compact: bool = False): ...
+    def write(self, path: PATH_OR_FILE) -> None: ...
+    def clear(self) -> None: ...
+    def setdefault(self, key: bytes, value: bytes) -> None: ...
+    def __delitem__(self, key: bytes) -> None: ...
+    def get(self, key: bytes, default: Any = ...) -> Any: ...
+    def pop(self, key: bytes, default: Any = ...) -> Any: ...
+    def __len__(self) -> int: ...
+    def size(self) -> int: ...
+    def compact(self) -> Any: ...
+
+class ChunkIndexEntry(NamedTuple):
+    refcount: int
+    size: int
+    csize: int
+
+CIE = Union[Tuple[int, int, int], Type[ChunkIndexEntry]]
+
+class ChunkKeyIterator:
+    def __init__(self, keysize: int) -> None: ...
+    def __iter__(self) -> Iterator: ...
+    def __next__(self) -> Tuple[bytes, Type[ChunkIndexEntry]]: ...
+
+class ChunkIndex(IndexBase):
+    def add(self, key: bytes, refs: int, size: int, csize: int) -> None: ...
+    def decref(self, key: bytes) -> CIE: ...
+    def incref(self, key: bytes) -> CIE: ...
+    def iteritems(self, marker: bytes = ...) -> Iterator: ...
+    def merge(self, other_index) -> None: ...
+    def stats_against(self, master_index) -> Tuple: ...
+    def summarize(self) -> Tuple: ...
+    def zero_csize_ids(self) -> int: ...
+    def __contains__(self, key: bytes) -> bool: ...
+    def __getitem__(self, key: bytes) -> Type[ChunkIndexEntry]: ...
+    def __setitem__(self, key: bytes, value: CIE) -> None: ...
+
+class NSIndexEntry(NamedTuple):
+    segment: int
+    offset: int
+    size: int
+
+class NSKeyIterator:
+    def __init__(self, keysize: int) -> None: ...
+    def __iter__(self) -> Iterator: ...
+    def __next__(self) -> Tuple[bytes, Type[Any]]: ...
+
+class NSIndex(IndexBase):
+    def iteritems(self, *args, **kwargs) -> Iterator: ...
+    def __contains__(self, key: bytes) -> bool: ...
+    def __getitem__(self, key: bytes) -> Any: ...
+    def __setitem__(self, key: bytes, value: Any) -> None: ...
+
+class NSIndex1(IndexBase):  # legacy
+    def iteritems(self, *args, **kwargs) -> Iterator: ...
+    def __contains__(self, key: bytes) -> bool: ...
+    def __getitem__(self, key: bytes) -> Any: ...
+    def __setitem__(self, key: bytes, value: Any) -> None: ...
+
+class FuseVersionsIndex(IndexBase):
+    def __contains__(self, key: bytes) -> bool: ...
+    def __getitem__(self, key: bytes) -> Any: ...
+    def __setitem__(self, key: bytes, value: Any) -> None: ...
+
+class CacheSynchronizer:
+    csize_parts: int
+    csize_totals: int
+    num_files_parts: int
+    num_files_totals: int
+    size_parts: int
+    size_totals: int
+    def __init__(self, chunks_index: Any) -> None: ...
+    def feed(self, chunk: bytes) -> None: ...

src/borg/item.pyi

@@ -0,0 +1,305 @@
+from typing import FrozenSet, Set, NamedTuple, Tuple, Mapping, Dict, List, Iterator, Callable, Any
+
+from .helpers import StableDict
+
+API_VERSION: str
+
+def want_bytes(v: Any, *, errors: str) -> bytes: ...
+def chunks_contents_equal(chunks1: Iterator, chunks2: Iterator) -> bool: ...
+
+class PropDict:
+    VALID_KEYS: Set[str] = ...
+    def __init__(self, data_dict: dict = None, internal_dict: dict = None, **kw) -> None: ...
+    def as_dict(self) -> StableDict: ...
+    def get(self, key: str, default: Any = None) -> Any: ...
+    def update(self, d: dict) -> None: ...
+    def update_internal(self, d: dict) -> None: ...
+    def __contains__(self, key: str) -> bool: ...
+    def __eq__(self, other: object) -> bool: ...
+
+class ArchiveItem(PropDict):
+    @property
+    def version(self) -> int: ...
+    @version.setter
+    def version(self, val: int) -> None: ...
+    @property
+    def name(self) -> str: ...
+    @name.setter
+    def name(self, val: str) -> None: ...
+    @property
+    def time(self) -> str: ...
+    @time.setter
+    def time(self, val: str) -> None: ...
+    @property
+    def time_end(self) -> str: ...
+    @time_end.setter
+    def time_end(self, val: str) -> None: ...
+    @property
+    def username(self) -> str: ...
+    @username.setter
+    def username(self, val: str) -> None: ...
+    @property
+    def hostname(self) -> str: ...
+    @hostname.setter
+    def hostname(self, val: str) -> None: ...
+    @property
+    def comment(self) -> str: ...
+    @comment.setter
+    def comment(self, val: str) -> None: ...
+    @property
+    def chunker_params(self) -> Tuple: ...
+    @chunker_params.setter
+    def chunker_params(self, val: Tuple) -> None: ...
+    @property
+    def cmdline(self) -> List[str]: ...
+    @cmdline.setter
+    def cmdline(self, val: List[str]) -> None: ...
+    @property
+    def recreate_cmdline(self) -> List[str]: ...
+    @recreate_cmdline.setter
+    def recreate_cmdline(self, val: List[str]) -> None: ...
+    @property
+    def recreate_args(self) -> Any: ...
+    @recreate_args.setter
+    def recreate_args(self, val: Any) -> None: ...
+    @property
+    def recreate_partial_chunks(self) -> Any: ...
+    @recreate_partial_chunks.setter
+    def recreate_partial_chunks(self, val: Any) -> None: ...
+    @property
+    def recreate_source_id(self) -> Any: ...
+    @recreate_source_id.setter
+    def recreate_source_id(self, val: Any) -> None: ...
+    @property
+    def nfiles(self) -> int: ...
+    @nfiles.setter
+    def nfiles(self, val: int) -> None: ...
+    @property
+    def nfiles_parts(self) -> int: ...
+    @nfiles_parts.setter
+    def nfiles_parts(self, val: int) -> None: ...
+    @property
+    def size(self) -> int: ...
+    @size.setter
+    def size(self, val: int) -> None: ...
+    @property
+    def size_parts(self) -> int: ...
+    @size_parts.setter
+    def size_parts(self, val: int) -> None: ...
+    @property
+    def csize(self) -> int: ...
+    @csize.setter
+    def csize(self, val: int) -> None: ...
+    @property
+    def csize_parts(self) -> int: ...
+    @csize_parts.setter
+    def csize_parts(self, val: int) -> None: ...
+    @property
+    def items(self) -> List: ...
+    @items.setter
+    def items(self, val: List) -> None: ...
+
+class ChunkListEntry(NamedTuple):
+    id: bytes
+    size: int
+    csize: int
+
+class Item(PropDict):
+    @property
+    def path(self) -> str: ...
+    @path.setter
+    def path(self, val: str) -> None: ...
+    @property
+    def source(self) -> str: ...
+    @source.setter
+    def source(self, val: str) -> None: ...
+    def is_dir(self) -> bool: ...
+    def is_link(self) -> bool: ...
+    def _is_type(self, typetest: Callable) -> bool: ...
+    @classmethod
+    def create_deleted(self, path) -> Item: ...
+    @classmethod
+    def from_optr(self, optr: Any) -> Item: ...
+    def to_optr(self) -> Any: ...
+    @property
+    def atime(self) -> int: ...
+    @atime.setter
+    def atime(self, val: int) -> None: ...
+    @property
+    def ctime(self) -> int: ...
+    @ctime.setter
+    def ctime(self, val: int) -> None: ...
+    @property
+    def mtime(self) -> int: ...
+    @mtime.setter
+    def mtime(self, val: int) -> None: ...
+    @property
+    def birthtime(self) -> int: ...
+    @birthtime.setter
+    def birthtime(self, val: int) -> None: ...
+    @property
+    def xattrs(self) -> StableDict: ...
+    @xattrs.setter
+    def xattrs(self, val: StableDict) -> None: ...
+    @property
+    def acl_access(self) -> bytes: ...
+    @acl_access.setter
+    def acl_access(self, val: bytes) -> None: ...
+    @property
+    def acl_default(self) -> bytes: ...
+    @acl_default.setter
+    def acl_default(self, val: bytes) -> None: ...
+    @property
+    def acl_extended(self) -> bytes: ...
+    @acl_extended.setter
+    def acl_extended(self, val: bytes) -> None: ...
+    @property
+    def acl_nfs4(self) -> bytes: ...
+    @acl_nfs4.setter
+    def acl_nfs4(self, val: bytes) -> None: ...
+    @property
+    def bsdflags(self) -> int: ...
+    @bsdflags.setter
+    def bsdflags(self, val: int) -> None: ...
+    @property
+    def chunks(self) -> List: ...
+    @chunks.setter
+    def chunks(self, val: List) -> None: ...
+    @property
+    def chunks_healthy(self) -> List: ...
+    @chunks_healthy.setter
+    def chunks_healthy(self, val: List) -> None: ...
+    @property
+    def deleted(self) -> bool: ...
+    @deleted.setter
+    def deleted(self, val: bool) -> None: ...
+    @property
+    def hardlink_master(self) -> bool: ...
+    @hardlink_master.setter
+    def hardlink_master(self, val: bool) -> None: ...
+    @property
+    def uid(self) -> int: ...
+    @uid.setter
+    def uid(self, val: int) -> None: ...
+    @property
+    def gid(self) -> int: ...
+    @gid.setter
+    def gid(self, val: int) -> None: ...
+    @property
+    def user(self) -> str: ...
+    @user.setter
+    def user(self, val: str) -> None: ...
+    @property
+    def group(self) -> str: ...
+    @group.setter
+    def group(self, val: str) -> None: ...
+    @property
+    def mode(self) -> int: ...
+    @mode.setter
+    def mode(self, val: int) -> None: ...
+    @property
+    def rdev(self) -> int: ...
+    @rdev.setter
+    def rdev(self, val: int) -> None: ...
+    @property
+    def nlink(self) -> int: ...
+    @nlink.setter
+    def nlink(self, val: int) -> None: ...
+    @property
+    def size(self) -> int: ...
+    @size.setter
+    def size(self, val: int) -> None: ...
+    def get_size(
+        self,
+        hardlink_masters=...,
+        memorize: bool = ...,
+        compressed: bool = ...,
+        from_chunks: bool = ...,
+        consider_ids: List[bytes] = ...,
+    ) -> int: ...
+    @property
+    def part(self) -> int: ...
+    @part.setter
+    def part(self, val: int) -> None: ...
+
+class ManifestItem(PropDict):
+    @property
+    def version(self) -> int: ...
+    @version.setter
+    def version(self, val: int) -> None: ...
+    @property
+    def timestamp(self) -> str: ...
+    @timestamp.setter
+    def timestamp(self, val: str) -> None: ...
+    @property
+    def archives(self) -> Mapping[bytes, dict]: ...
+    @archives.setter
+    def archives(self, val: Mapping[bytes, dict]) -> None: ...
+    @property
+    def config(self) -> Dict: ...
+    @config.setter
+    def config(self, val: Dict) -> None: ...
+    @property
+    def item_keys(self) -> Tuple: ...
+    @item_keys.setter
+    def item_keys(self, val: Tuple) -> None: ...
+
+class ItemDiff:
+    def __init__(self, *args, **kwargs) -> None: ...
+    def _chunk_content_equal(self, c1: Iterator, c2: Iterator) -> bool: ...
+
+class Key(PropDict):
+    @property
+    def version(self) -> int: ...
+    @version.setter
+    def version(self, val: int) -> None: ...
+    @property
+    def chunk_seed(self) -> int: ...
+    @chunk_seed.setter
+    def chunk_seed(self, val: int) -> None: ...
+    @property
+    def tam_required(self) -> bool: ...
+    @tam_required.setter
+    def tam_required(self, val: bool) -> None: ...
+    @property
+    def enc_hmac_key(self) -> bytes: ...
+    @enc_hmac_key.setter
+    def enc_hmac_key(self, val: bytes) -> None: ...
+    @property
+    def enc_key(self) -> bytes: ...
+    @enc_key.setter
+    def enc_key(self, val: bytes) -> None: ...
+    @property
+    def id_key(self) -> bytes: ...
+    @id_key.setter
+    def id_key(self, val: bytes) -> None: ...
+    @property
+    def repository_id(self) -> bytes: ...
+    @repository_id.setter
+    def repository_id(self, val: bytes) -> None: ...
+
+class EncryptedKey(PropDict):
+    @property
+    def version(self) -> int: ...
+    @version.setter
+    def version(self, val: int) -> None: ...
+    @property
+    def algorithm(self) -> str: ...
+    @algorithm.setter
+    def algorithm(self, val: str) -> None: ...
+    @property
+    def salt(self) -> bytes: ...
+    @salt.setter
+    def salt(self, val: bytes) -> None: ...
+    @property
+    def iterations(self) -> int: ...
+    @iterations.setter
+    def iterations(self, val: int) -> None: ...
+    @property
+    def data(self) -> bytes: ...
+    @data.setter
+    def data(self, val: bytes) -> None: ...
+    @property
+    def hash(self) -> bytes: ...
+    @hash.setter
+    def hash(self, val: bytes) -> None: ...

tox.ini

@@ -29,3 +29,11 @@ changedir =
 deps =
     flake8
 commands = flake8 src scripts conftest.py
+
+[testenv:mypy]
+changedir =
+deps =
+    pytest
+    mypy
+    pkgconfig
+commands = mypy