mypy: fixes / annotations

src/borg/__init__.py

@@ -4,7 +4,7 @@ from ._version import version as __version__
 
 
 _v = parse_version(__version__)
-__version_tuple__ = _v._version.release
+__version_tuple__ = _v._version.release  # type: ignore
 
 # assert that all semver components are integers
 # this is mainly to show errors when people repackage poorly

src/borg/cache.py

@@ -1109,7 +1109,7 @@ Chunk index:    {0.total_unique_chunks:20d}             unknown"""
     def __exit__(self, exc_type, exc_val, exc_tb):
         pass
 
-    files = None
+    files = None  # type: ignore
     cache_mode = "d"
 
     def file_known_and_unchanged(self, hashed_path, path_hash, st):

src/borg/crypto/file_integrity.py

@@ -3,6 +3,7 @@ import io
 import json
 import os
 from hmac import compare_digest
+from typing import Callable
 
 from ..helpers import IntegrityError
 from ..logger import create_logger
@@ -54,8 +55,8 @@ class FileHashingWrapper(FileLikeWrapper):
     are illegal.
     """
 
-    ALGORITHM = None
-    FACTORY = None
+    ALGORITHM: str = None
+    FACTORY: Callable = None
 
     def __init__(self, backing_fd, write):
         self.fd = backing_fd

src/borg/crypto/key.py

@@ -3,7 +3,7 @@ import os
 import textwrap
 from binascii import a2b_base64, b2a_base64, hexlify
 from hashlib import sha256, pbkdf2_hmac
-from typing import Literal
+from typing import Literal, Callable, Sequence
 
 from ..logger import create_logger
 
@@ -139,9 +139,9 @@ def uses_same_id_hash(other_key, key):
 
 class KeyBase:
     # Numeric key type ID, must fit in one byte.
-    TYPE = None  # override in subclasses
+    TYPE: int = None  # override in subclasses
     # set of key type IDs the class can handle as input
-    TYPES_ACCEPTABLE = None  # override in subclasses
+    TYPES_ACCEPTABLE: set[int] = None  # override in subclasses
 
     # Human-readable name
     NAME = "UNDEFINED"
@@ -154,7 +154,7 @@ class KeyBase:
 
     # Seed for the buzhash chunker (borg.algorithms.chunker.Chunker)
     # type is int
-    chunk_seed = None
+    chunk_seed: int = None
 
     # Whether this *particular instance* is encrypted from a practical point of view,
     # i.e. when it's using encryption with a empty passphrase, then
@@ -356,7 +356,7 @@ class AESKeyBase(KeyBase):
 
     PAYLOAD_OVERHEAD = 1 + 32 + 8  # TYPE + HMAC + NONCE
 
-    CIPHERSUITE = None  # override in derived class
+    CIPHERSUITE: Callable = None  # override in derived class
 
     logically_encrypted = True
 
@@ -839,7 +839,7 @@ class AEADKeyBase(KeyBase):
 
     PAYLOAD_OVERHEAD = 1 + 1 + 6 + 24 + 16  # [bytes], see Layout
 
-    CIPHERSUITE = None  # override in subclass
+    CIPHERSUITE: Callable = None  # override in subclass
 
     logically_encrypted = True
 

src/borg/fuse.py

@@ -706,7 +706,7 @@ class FuseOperations(llfuse.Operations, FuseBackend):
     # note: we can't have a generator (with yield) and not a generator (async) in the same method
     if has_pyfuse3:
 
-        async def readdir(self, fh, off, token):
+        async def readdir(self, fh, off, token):  # type: ignore[misc]
             entries = [(b".", fh), (b"..", self.parent[fh])]
             entries.extend(self.contents[fh].items())
             for i, (name, inode) in enumerate(entries[off:], off):
@@ -716,7 +716,7 @@ class FuseOperations(llfuse.Operations, FuseBackend):
 
     else:
 
-        def readdir(self, fh, off):
+        def readdir(self, fh, off):  # type: ignore[misc]
             entries = [(b".", fh), (b"..", self.parent[fh])]
             entries.extend(self.contents[fh].items())
             for i, (name, inode) in enumerate(entries[off:], off):

src/borg/helpers/__init__.py

@@ -5,18 +5,38 @@ that did not fit better elsewhere.
 Code used to be in borg/helpers.py but was split into the modules in this
 package, which are imported into here for compatibility.
 """
+import os
 
-from .checks import *  # NOQA
-from .datastruct import *  # NOQA
-from .errors import *  # NOQA
-from .fs import *  # NOQA
-from .manifest import *  # NOQA
-from .misc import *  # NOQA
-from .parseformat import *  # NOQA
-from .process import *  # NOQA
-from .progress import *  # NOQA
-from .time import *  # NOQA
-from .yes_no import *  # NOQA
+from ..constants import *  # NOQA
+from .checks import check_extension_modules, check_python
+from .datastruct import StableDict, Buffer, EfficientCollectionQueue
+from .errors import Error, ErrorWithTraceback, IntegrityError, DecompressionError
+from .fs import ensure_dir, get_security_dir, get_keys_dir, get_base_dir, get_cache_dir, get_config_dir
+from .fs import dir_is_tagged, dir_is_cachedir, make_path_safe, scandir_inorder
+from .fs import secure_erase, safe_unlink, dash_open, os_open, os_stat, umount
+from .fs import O_, flags_root, flags_dir, flags_special_follow, flags_special, flags_base, flags_normal, flags_noatime
+from .fs import HardLinkManager
+from .manifest import Manifest, NoManifestError, MandatoryFeatureUnsupported, AI_HUMAN_SORT_KEYS
+from .misc import prune_within, prune_split, PRUNING_PATTERNS, sysinfo, log_multi, consume, get_tar_filter
+from .misc import ChunkIteratorFileWrapper, open_item, chunkit, iter_separated, ErrorIgnoringTextIOWrapper
+from .parseformat import bin_to_hex, safe_encode, safe_decode
+from .parseformat import remove_surrogates, eval_escapes, decode_dict, positive_int_validator, interval
+from .parseformat import ChunkerParams, FilesCacheMode, partial_format, DatetimeWrapper
+from .parseformat import format_file_size, parse_file_size, FileSize, parse_storage_quota
+from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal
+from .parseformat import format_line, replace_placeholders, PlaceholderError
+from .parseformat import PrefixSpec, GlobSpec, CommentSpec, SortBySpec, NameSpec
+from .parseformat import format_archive, parse_stringified_list, clean_lines
+from .parseformat import Location, location_validator, archivename_validator
+from .parseformat import BaseFormatter, ArchiveFormatter, ItemFormatter, file_status
+from .parseformat import swidth_slice, ellipsis_truncate
+from .parseformat import BorgJsonEncoder, basic_json_data, json_print, json_dump, prepare_dump_dict
+from .process import daemonize, daemonizing, signal_handler, raising_signal_handler, sig_int, SigHup, SigTerm
+from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process
+from .progress import ProgressIndicatorPercent, ProgressIndicatorEndless, ProgressIndicatorMessage
+from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS
+from .time import format_time, format_timedelta, isoformat_time, to_localtime, OutputTimestamp
+from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH
 
 from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker
 from . import msgpack

src/borg/helpers/manifest.py

@@ -5,6 +5,7 @@ import re
 from collections import abc, namedtuple
 from datetime import datetime, timedelta
 from operator import attrgetter
+from typing import Sequence, FrozenSet
 
 from .errors import Error
 
@@ -158,9 +159,9 @@ class Manifest:
         # count and the need to be able to find all (directly and indirectly) referenced chunks of a given archive.
         DELETE = "delete"
 
-    NO_OPERATION_CHECK = tuple()
+    NO_OPERATION_CHECK: Sequence[Operation] = tuple()
 
-    SUPPORTED_REPO_FEATURES = frozenset([])
+    SUPPORTED_REPO_FEATURES: FrozenSet[str] = frozenset([])
 
     MANIFEST_ID = b"\0" * 32
 

src/borg/helpers/misc.py

@@ -236,12 +236,12 @@ def iter_separated(fd, sep=None, read_size=4096):
     sep = sep or ("\n" if is_str else b"\n")
     while len(buf) > 0:
         part2, *items = buf.split(sep)
-        *full, part = (part + part2, *items)
+        *full, part = (part + part2, *items)  # type: ignore
         yield from full
         buf = fd.read(read_size)
     # won't yield an empty part if stream ended with `sep`
     # or if there was no data before EOF
-    if len(part) > 0:
+    if len(part) > 0:  # type: ignore[arg-type]
         yield part
 
 

src/borg/helpers/parseformat.py

@@ -694,7 +694,7 @@ class ArchiveFormatter(BaseFormatter):
 class ItemFormatter(BaseFormatter):
     # we provide the hash algos from python stdlib (except shake_*) and additionally xxh64.
     # shake_* is not provided because it uses an incompatible .digest() method to support variable length.
-    hash_algorithms = hashlib.algorithms_guaranteed.union({"xxh64"}).difference({"shake_128", "shake_256"})
+    hash_algorithms = set(hashlib.algorithms_guaranteed).union({"xxh64"}).difference({"shake_128", "shake_256"})
     KEY_DESCRIPTIONS = {
         "bpath": "verbatim POSIX path, can contain any character except NUL",
         "path": "path interpreted as text (might be missing non-text characters, see bpath)",

src/borg/helpers/progress.py

@@ -21,7 +21,7 @@ def justify_to_terminal_size(message):
 
 class ProgressIndicatorBase:
     LOGGER = "borg.output.progress"
-    JSON_TYPE = None
+    JSON_TYPE: str = None
     json = False
 
     operation_id_counter = 0

src/borg/logger.py

@@ -135,7 +135,64 @@ def find_parent_module():
         return __name__
 
 
-def create_logger(name=None):
+class LazyLogger:
+    def __init__(self, name=None):
+        self.__name = name or find_parent_module()
+        self.__real_logger = None
+
+    @property
+    def __logger(self):
+        if self.__real_logger is None:
+            if not configured:
+                raise Exception("tried to call a logger before setup_logging() was called")
+            self.__real_logger = logging.getLogger(self.__name)
+            if self.__name.startswith("borg.debug.") and self.__real_logger.level == logging.NOTSET:
+                self.__real_logger.setLevel("WARNING")
+        return self.__real_logger
+
+    def getChild(self, suffix):
+        return LazyLogger(self.__name + "." + suffix)
+
+    def setLevel(self, *args, **kw):
+        return self.__logger.setLevel(*args, **kw)
+
+    def log(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.log(*args, **kw)
+
+    def exception(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.exception(*args, **kw)
+
+    def debug(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.debug(*args, **kw)
+
+    def info(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.info(*args, **kw)
+
+    def warning(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.warning(*args, **kw)
+
+    def error(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.error(*args, **kw)
+
+    def critical(self, *args, **kw):
+        if "msgid" in kw:
+            kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
+        return self.__logger.critical(*args, **kw)
+
+
+def create_logger(name: str = None) -> LazyLogger:
     """lazily create a Logger object with the proper path, which is returned by
     find_parent_module() by default, or is provided via the commandline
 
@@ -152,62 +209,6 @@ def create_logger(name=None):
     If you try, you'll get an exception.
     """
 
-    class LazyLogger:
-        def __init__(self, name=None):
-            self.__name = name or find_parent_module()
-            self.__real_logger = None
-
-        @property
-        def __logger(self):
-            if self.__real_logger is None:
-                if not configured:
-                    raise Exception("tried to call a logger before setup_logging() was called")
-                self.__real_logger = logging.getLogger(self.__name)
-                if self.__name.startswith("borg.debug.") and self.__real_logger.level == logging.NOTSET:
-                    self.__real_logger.setLevel("WARNING")
-            return self.__real_logger
-
-        def getChild(self, suffix):
-            return LazyLogger(self.__name + "." + suffix)
-
-        def setLevel(self, *args, **kw):
-            return self.__logger.setLevel(*args, **kw)
-
-        def log(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.log(*args, **kw)
-
-        def exception(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.exception(*args, **kw)
-
-        def debug(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.debug(*args, **kw)
-
-        def info(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.info(*args, **kw)
-
-        def warning(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.warning(*args, **kw)
-
-        def error(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.error(*args, **kw)
-
-        def critical(self, *args, **kw):
-            if "msgid" in kw:
-                kw.setdefault("extra", {})["msgid"] = kw.pop("msgid")
-            return self.__logger.critical(*args, **kw)
-
     return LazyLogger(name)
 
 

src/borg/patterns.py

@@ -172,7 +172,7 @@ def normalize_path(path):
 class PatternBase:
     """Shared logic for inclusion/exclusion patterns."""
 
-    PREFIX = NotImplemented
+    PREFIX: str = None
 
     def __init__(self, pattern, recurse_dir=False):
         self.pattern_orig = pattern

src/borg/platform/base.py

@@ -82,7 +82,7 @@ def acl_set(path, item, numeric_ids=False, fd=None):
 
 
 try:
-    from os import lchflags
+    from os import lchflags  # type: ignore[attr-defined]
 
     def set_flags(path, bsd_flags, fd=None):
         lchflags(path, bsd_flags)

src/borg/remote.py

@@ -495,7 +495,7 @@ def api(*, since, **kwargs_decorator):
 
 
 class RemoteRepository:
-    extra_test_args = []
+    extra_test_args = []  # type: ignore
 
     class RPCError(Exception):
         def __init__(self, unpacked):

src/borg/testsuite/__init__.py

@@ -168,7 +168,7 @@ class BaseTestCase(unittest.TestCase):
     if raises:
         assert_raises = staticmethod(raises)
     else:
-        assert_raises = unittest.TestCase.assertRaises
+        assert_raises = unittest.TestCase.assertRaises  # type: ignore
 
     @contextmanager
     def assert_creates_file(self, path):

src/borg/testsuite/archiver.py

@@ -250,7 +250,7 @@ def test_disk_full(cmd):
 
 
 class ArchiverTestCaseBase(BaseTestCase):
-    EXE = None  # python source based
+    EXE: str = None  # python source based
     FORK_DEFAULT = False
     prefix = ""
 

src/borg/testsuite/platform.py

@@ -38,7 +38,7 @@ other::r--
     "ascii"
 )
 
-_acls_working = None
+# _acls_working = None
 
 
 def fakeroot_detected():

src/borg/testsuite/remote.py

@@ -113,9 +113,9 @@ class TestRepositoryCache:
 
         # Force cache to back off
         qsl = cache.query_size_limit
-        cache.query_size_limit = query_size_limit
+        cache.query_size_limit = query_size_limit  # type: ignore[assignment]
         cache.backoff()
-        cache.query_size_limit = qsl
+        cache.query_size_limit = qsl  # type: ignore[assignment]
         # Evicted H(1) and H(2)
         assert cache.evictions == 2
         assert H(1) not in cache.cache

tox.ini

@@ -36,4 +36,5 @@ deps =
     pytest
     mypy
     pkgconfig
+    types-python-dateutil
 commands = mypy