mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #7650 from ThomasWaldmann/remove-force-tam-not-required 

TAM: always generate and verify
This commit is contained in:
TW 2023-09-04 20:33:52 +02:00 committed by GitHub
parent 4ded3620c5 a0f5264cbd
commit 1fd8b52f6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 35 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/borg/archive.py
View File

@ -493,7 +493,6 @@ class Archive:
self.name = name # overwritten later with name from archive metadata
self.name_in_manifest = name # can differ from .name later (if borg check fixed duplicate archive names)
self.comment = None
self.tam_verified = False
self.numeric_ids = numeric_ids
self.noatime = noatime
self.noctime = noctime
@ -533,8 +532,7 @@ class Archive:
def _load_meta(self, id):
cdata = self.repository.get(id)
_, data = self.repo_objs.parse(id, cdata)
# we do not require TAM for archives, otherwise we can not even borg list a repo with old archives.
archive, self.tam_verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=True)
archive, _ = self.key.unpack_and_verify_archive(data)
metadata = ArchiveItem(internal_dict=archive)
if metadata.version not in (1, 2): # legacy: still need to read v1 archives
raise Exception("Unknown archive metadata version")
@ -1998,7 +1996,7 @@ class ArchiveChecker:
# **after** doing the low-level checks and having a strong indication that we
# are likely looking at an archive item here, also check the TAM authentication:
try:
archive, verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=False)
archive, _ = self.key.unpack_and_verify_archive(data)
except IntegrityError as integrity_error:
# TAM issues - do not accept this archive!
# either somebody is trying to attack us with a fake archive data or
@ -2269,7 +2267,7 @@ class ArchiveChecker:
del self.manifest.archives[info.name]
continue
try:
archive, verified, salt = self.key.unpack_and_verify_archive(data, force_tam_not_required=False)
archive, salt = self.key.unpack_and_verify_archive(data)
except IntegrityError as integrity_error:
# looks like there is a TAM issue with this archive, this might be an attack!
# when upgrading to borg 1.2.5, users are expected to TAM-authenticate all archives they

2
src/borg/archiver/key_cmds.py
View File

@ -63,7 +63,7 @@ class KeysMixIn:
print("Change not needed or not supported.")
return EXIT_WARNING
for name in ("repository_id", "crypt_key", "id_key", "chunk_seed", "tam_required", "sessionid", "cipher"):
for name in ("repository_id", "crypt_key", "id_key", "chunk_seed", "sessionid", "cipher"):
value = getattr(key, name)
setattr(key_new, name, value)

6
src/borg/archiver/rcreate_cmd.py
View File

@ -3,7 +3,7 @@ import argparse
from ._common import with_repository, with_other_repository, Highlander
from ..cache import Cache
from ..constants import * # NOQA
from ..crypto.key import key_creator, key_argument_names, tam_required_file
from ..crypto.key import key_creator, key_argument_names
from ..helpers import EXIT_WARNING
from ..helpers import location_validator, Location
from ..helpers import parse_storage_quota
@ -35,10 +35,6 @@ class RCreateMixIn:
repository.commit(compact=False)
with Cache(repository, manifest, warn_if_unencrypted=False):
pass
if key.tam_required:
tam_file = tam_required_file(repository)
open(tam_file, "w").close()
if key.NAME != "plaintext":
logger.warning(
"\n"

2
src/borg/cache.py
View File

@ -755,7 +755,7 @@ class LocalCache(CacheStatsMixin):
nonlocal processed_item_metadata_chunks
csize, data = decrypted_repository.get(archive_id)
chunk_idx.add(archive_id, 1, len(data))
archive, verified, _ = self.key.unpack_and_verify_archive(data, force_tam_not_required=True)
archive, _ = self.key.unpack_and_verify_archive(data)
archive = ArchiveItem(internal_dict=archive)
if archive.version not in (1, 2): # legacy
raise Exception("Unknown archive metadata version")

73
src/borg/crypto/key.py
View File

@ -15,7 +15,7 @@ import argon2.low_level
from ..constants import * # NOQA
from ..helpers import StableDict
from ..helpers import Error, IntegrityError
from ..helpers import get_keys_dir, get_security_dir
from ..helpers import get_keys_dir
from ..helpers import get_limited_unpacker
from ..helpers import bin_to_hex
from ..helpers.passphrase import Passphrase, PasswordRetriesExceeded, PassphraseWrong
@ -136,16 +136,6 @@ def key_factory(repository, manifest_chunk, *, ro_cls=RepoObj):
return identify_key(manifest_data).detect(repository, manifest_data)
def tam_required_file(repository):
security_dir = get_security_dir(bin_to_hex(repository.id), legacy=(repository.version == 1))
return os.path.join(security_dir, "tam_required")
def tam_required(repository):
file = tam_required_file(repository)
return os.path.isfile(file)
def uses_same_chunker_secret(other_key, key):
"""is the chunker secret the same?"""
# avoid breaking the deduplication by a different chunker secret
@ -211,7 +201,6 @@ class KeyBase:
self.TYPE_STR = bytes([self.TYPE])
self.repository = repository
self.target = None # key location file path / repo obj
self.tam_required = True
self.copy_crypt_key = False
def id_hash(self, data):
@ -253,39 +242,25 @@ class KeyBase:
tam["hmac"] = hmac.digest(tam_key, packed, "sha512")
return msgpack.packb(metadata_dict)
def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
"""Unpack msgpacked *data* and return (object, did_verify)."""
def unpack_and_verify_manifest(self, data):
"""Unpack msgpacked *data* and return manifest."""
if data.startswith(b"\xc1" * 4):
# This is a manifest from the future, we can't read it.
raise UnsupportedManifestError()
tam_required = self.tam_required
if force_tam_not_required and tam_required:
logger.warning("Manifest authentication DISABLED.")
tam_required = False
data = bytearray(data)
unpacker = get_limited_unpacker("manifest")
unpacker.feed(data)
unpacked = unpacker.unpack()
if AUTHENTICATED_NO_KEY:
return unpacked, True # True is a lie.
return unpacked
if "tam" not in unpacked:
if tam_required:
raise TAMRequiredError(self.repository._location.canonical_path())
else:
logger.debug("Manifest TAM not found and not required")
return unpacked, False
raise TAMRequiredError(self.repository._location.canonical_path())
tam = unpacked.pop("tam", None)
if not isinstance(tam, dict):
raise TAMInvalid()
tam_type = tam.get("type", "<none>")
if tam_type != "HKDF_HMAC_SHA512":
if tam_required:
raise TAMUnsupportedSuiteError(repr(tam_type))
else:
logger.debug(
"Ignoring manifest TAM made with unsupported suite, since TAM is not required: %r", tam_type
)
return unpacked, False
raise TAMUnsupportedSuiteError(repr(tam_type))
tam_hmac = tam.get("hmac")
tam_salt = tam.get("salt")
if not isinstance(tam_salt, (bytes, str)) or not isinstance(tam_hmac, (bytes, str)):
@ -299,39 +274,23 @@ class KeyBase:
if not hmac.compare_digest(calculated_hmac, tam_hmac):
raise TAMInvalid()
logger.debug("TAM-verified manifest")
return unpacked, True
return unpacked
def unpack_and_verify_archive(self, data, force_tam_not_required=False):
"""Unpack msgpacked *data* and return (object, did_verify)."""
tam_required = self.tam_required
if force_tam_not_required and tam_required:
# for a long time, borg only checked manifest for "tam_required" and
# people might have archives without TAM, so don't be too annoyingly loud here:
logger.debug("Archive authentication DISABLED.")
tam_required = False
def unpack_and_verify_archive(self, data):
"""Unpack msgpacked *data* and return (object, salt)."""
data = bytearray(data)
unpacker = get_limited_unpacker("archive")
unpacker.feed(data)
unpacked = unpacker.unpack()
if "tam" not in unpacked:
if tam_required:
archive_name = unpacked.get("name", "<unknown>")
raise ArchiveTAMRequiredError(archive_name)
else:
logger.debug("Archive TAM not found and not required")
return unpacked, False, None
archive_name = unpacked.get("name", "<unknown>")
raise ArchiveTAMRequiredError(archive_name)
tam = unpacked.pop("tam", None)
if not isinstance(tam, dict):
raise ArchiveTAMInvalid()
tam_type = tam.get("type", "<none>")
if tam_type != "HKDF_HMAC_SHA512":
if tam_required:
raise TAMUnsupportedSuiteError(repr(tam_type))
else:
logger.debug(
"Ignoring archive TAM made with unsupported suite, since TAM is not required: %r", tam_type
)
return unpacked, False, None
raise TAMUnsupportedSuiteError(repr(tam_type))
tam_hmac = tam.get("hmac")
tam_salt = tam.get("salt")
if not isinstance(tam_salt, (bytes, str)) or not isinstance(tam_hmac, (bytes, str)):
@ -345,7 +304,7 @@ class KeyBase:
if not hmac.compare_digest(calculated_hmac, tam_hmac):
raise ArchiveTAMInvalid()
logger.debug("TAM-verified archive")
return unpacked, True, tam_salt
return unpacked, tam_salt
class PlaintextKey(KeyBase):
@ -357,10 +316,6 @@ class PlaintextKey(KeyBase):
chunk_seed = 0
logically_encrypted = False
def __init__(self, repository):
super().__init__(repository)
self.tam_required = False
@classmethod
def create(cls, repository, args, **kw):
logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.')
@ -526,7 +481,6 @@ class FlexiKey:
self.crypt_key = key.crypt_key
self.id_key = key.id_key
self.chunk_seed = key.chunk_seed
self.tam_required = key.get("tam_required", tam_required(self.repository))
return True
return False
@ -639,7 +593,6 @@ class FlexiKey:
crypt_key=self.crypt_key,
id_key=self.id_key,
chunk_seed=self.chunk_seed,
tam_required=self.tam_required,
)
data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm)
key_data = "\n".join(textwrap.wrap(b2a_base64(data).decode("ascii")))

4
src/borg/helpers/parseformat.py
View File

@ -751,7 +751,6 @@ class ArchiveFormatter(BaseFormatter):
"username": partial(self.get_meta, "username", ""),
"comment": partial(self.get_meta, "comment", ""),
"command_line": partial(self.get_meta, "command_line", ""),
"tam": self.get_tam,
"size": partial(self.get_meta, "size", 0),
"nfiles": partial(self.get_meta, "nfiles", 0),
"end": self.get_ts_end,
@ -797,9 +796,6 @@ class ArchiveFormatter(BaseFormatter):
def get_ts_end(self):
return self.format_time(self.archive.ts_end)
def get_tam(self):
return "verified" if self.archive.tam_verified else "none"
def format_time(self, ts):
return OutputTimestamp(ts)

3
src/borg/item.pyx
View File

@ -467,7 +467,7 @@ cdef class Key(PropDict):
crypt_key = PropDictProperty(bytes)
id_key = PropDictProperty(bytes)
chunk_seed = PropDictProperty(int)
tam_required = PropDictProperty(bool)
tam_required = PropDictProperty(bool) # legacy. borg now implicitly always requires TAM.
def update_internal(self, d):
# legacy support for migration (data from old msgpacks comes in as bytes always, but sometimes we want str)
@ -650,7 +650,6 @@ class ItemDiff:
self._can_compare_chunk_ids = can_compare_chunk_ids
self._chunk_1 = chunk_1
self._chunk_2 = chunk_2
self._changes = {}
if self._item1.is_link() or self._item2.is_link():

25
src/borg/manifest.py
View File

@ -1,6 +1,4 @@
import enum
import os
import os.path
import re
from collections import abc, namedtuple
from datetime import datetime, timedelta, timezone
@ -229,7 +227,6 @@ class Manifest:
self.repo_objs = ro_cls(key)
self.repository = repository
self.item_keys = frozenset(item_keys) if item_keys is not None else ITEM_KEYS
self.tam_verified = False
self.timestamp = None
@property
@ -241,9 +238,9 @@ class Manifest:
return parse_timestamp(self.timestamp)
@classmethod
def load(cls, repository, operations, key=None, force_tam_not_required=False, *, ro_cls=RepoObj):
def load(cls, repository, operations, key=None, *, ro_cls=RepoObj):
from .item import ManifestItem
from .crypto.key import key_factory, tam_required_file, tam_required
from .crypto.key import key_factory
from .repository import Repository
try:
@ -254,9 +251,7 @@ class Manifest:
key = key_factory(repository, cdata, ro_cls=ro_cls)
manifest = cls(key, repository, ro_cls=ro_cls)
_, data = manifest.repo_objs.parse(cls.MANIFEST_ID, cdata)
manifest_dict, manifest.tam_verified = key.unpack_and_verify_manifest(
data, force_tam_not_required=force_tam_not_required
)
manifest_dict = key.unpack_and_verify_manifest(data)
m = ManifestItem(internal_dict=manifest_dict)
manifest.id = manifest.repo_objs.id_hash(data)
if m.get("version") not in (1, 2):
@ -268,17 +263,6 @@ class Manifest:
manifest.item_keys = ITEM_KEYS
manifest.item_keys |= frozenset(m.config.get("item_keys", [])) # new location of item_keys since borg2
manifest.item_keys |= frozenset(m.get("item_keys", [])) # legacy: borg 1.x: item_keys not in config yet
if manifest.tam_verified:
manifest_required = manifest.config.get("tam_required", False)
security_required = tam_required(repository)
if manifest_required and not security_required:
logger.debug("Manifest is TAM verified and says TAM is required, updating security database...")
file = tam_required_file(repository)
open(file, "w").close()
if not manifest_required and security_required:
logger.debug("Manifest is TAM verified and says TAM is *not* required, updating security database...")
os.unlink(tam_required_file(repository))
manifest.check_repository_compatibility(operations)
return manifest
@ -310,8 +294,6 @@ class Manifest:
def write(self):
from .item import ManifestItem
if self.key.tam_required:
self.config["tam_required"] = True
# self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
if self.timestamp is None:
self.timestamp = datetime.now(tz=timezone.utc).isoformat(timespec="microseconds")
@ -331,7 +313,6 @@ class Manifest:
timestamp=self.timestamp,
config=StableDict(self.config),
)
self.tam_verified = True
data = self.key.pack_and_authenticate_metadata(manifest.as_dict())
self.id = self.repo_objs.id_hash(data)
self.repository.put(self.MANIFEST_ID, self.repo_objs.format(self.MANIFEST_ID, {}, data))

11
src/borg/testsuite/archiver/checks.py
View File

@ -413,8 +413,8 @@ def test_check_rebuild_manifest(archiver):
repository.delete(Manifest.MANIFEST_ID) # kill manifest, so check has to rebuild it
repository.commit(compact=False)
cmd(archiver, "check", "--repair")
output = cmd(archiver, "rlist", "--format='{name} tam:{tam}{NL}'")
assert "archive_tam tam:verified" in output # TAM-verified archive is in rebuilt manifest
output = cmd(archiver, "rlist", "--format='{name}{NL}'")
assert "archive_tam" in output # TAM-verified archive is in rebuilt manifest
assert "archive_no_tam" not in output # check got rid of untrusted not TAM-verified archive
@ -425,12 +425,9 @@ def test_check_rebuild_refcounts(archiver):
repository = Repository(archiver.repository_path, exclusive=True)
with repository:
write_archive_without_tam(repository, "archive_no_tam")
output = cmd(archiver, "rlist", "--format='{name} tam:{tam}{NL}'")
assert "archive_tam tam:verified" in output # good
assert "archive_no_tam tam:none" in output # could be borg < 1.0.9 archive or fake
cmd(archiver, "check", "--repair")
output = cmd(archiver, "rlist", "--format='{name} tam:{tam}{NL}'")
assert "archive_tam tam:verified" in output # TAM-verified archive still there
output = cmd(archiver, "rlist", "--format='{name}{NL}'")
assert "archive_tam" in output # TAM-verified archive still there
assert "archive_no_tam" not in output # check got rid of untrusted not TAM-verified archive
archive_id_post_check = cmd(archiver, "rlist", "--format='{name} {id}{NL}'")
assert archive_id_post_check == archive_id_pre_check # rebuild_refcounts didn't change archive_tam archive id

38
src/borg/testsuite/key.py
View File

@ -272,39 +272,19 @@ class TestTAM:
with pytest.raises(msgpack.UnpackException):
key.unpack_and_verify_manifest(blob)
def test_missing_when_required(self, key):
blob = msgpack.packb({})
with pytest.raises(TAMRequiredError):
key.unpack_and_verify_manifest(blob)
with pytest.raises(TAMRequiredError):
key.unpack_and_verify_archive(blob)
def test_missing(self, key):
blob = msgpack.packb({})
key.tam_required = False
unpacked, verified = key.unpack_and_verify_manifest(blob)
assert unpacked == {}
assert not verified
unpacked, verified, _ = key.unpack_and_verify_archive(blob)
assert unpacked == {}
assert not verified
def test_unknown_type_when_required(self, key):
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
with pytest.raises(TAMUnsupportedSuiteError):
with pytest.raises(TAMRequiredError):
key.unpack_and_verify_manifest(blob)
with pytest.raises(TAMUnsupportedSuiteError):
with pytest.raises(TAMRequiredError):
key.unpack_and_verify_archive(blob)
def test_unknown_type(self, key):
blob = msgpack.packb({"tam": {"type": "HMAC_VOLLBIT"}})
key.tam_required = False
unpacked, verified = key.unpack_and_verify_manifest(blob)
assert unpacked == {}
assert not verified
unpacked, verified, _ = key.unpack_and_verify_archive(blob)
assert unpacked == {}
assert not verified
with pytest.raises(TAMUnsupportedSuiteError):
key.unpack_and_verify_manifest(blob)
with pytest.raises(TAMUnsupportedSuiteError):
key.unpack_and_verify_archive(blob)
@pytest.mark.parametrize(
"tam, exc",
@ -360,8 +340,7 @@ class TestTAM:
unpacked = msgpack.unpackb(blob)
assert unpacked["tam"]["type"] == "HKDF_HMAC_SHA512"
unpacked, verified = key.unpack_and_verify_manifest(blob)
assert verified
unpacked = key.unpack_and_verify_manifest(blob)
assert unpacked["foo"] == "bar"
assert "tam" not in unpacked
@ -373,8 +352,7 @@ class TestTAM:
unpacked = msgpack.unpackb(blob)
assert unpacked["tam"]["type"] == "HKDF_HMAC_SHA512"
unpacked, verified, _ = key.unpack_and_verify_archive(blob)
assert verified
unpacked, _ = key.unpack_and_verify_archive(blob)
assert unpacked["foo"] == "bar"
assert "tam" not in unpacked