mirror of
https://github.com/borgbackup/borg.git
synced 2025-03-04 10:39:50 +00:00
Merge pull request #7556 from ThomasWaldmann/kill-nonce-manager
remove nonce management, related repo methods
This commit is contained in:
TW
GitHub
commit
b70cefa474
7 changed files with 2 additions and 381 deletions
|
|
@ -25,7 +25,6 @@ from ..platform import SaveFile
|
|||
from ..repoobj import RepoObj
|
||||
|
||||
|
||||
from .nonces import NonceManager
|
||||
from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
|
||||
from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305
|
||||
from . import low_level
|
||||
|
|
@ -372,7 +371,8 @@ class AESKeyBase(KeyBase):
|
|||
logically_encrypted = True
|
||||
|
||||
def encrypt(self, id, data):
|
||||
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), self.cipher.block_count(len(data)))
|
||||
# legacy, this is only used by the tests.
|
||||
next_iv = self.cipher.next_iv()
|
||||
return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)
|
||||
|
||||
def decrypt(self, id, data):
|
||||
|
|
@ -411,7 +411,6 @@ class AESKeyBase(KeyBase):
|
|||
manifest_blocks = num_cipher_blocks(len(manifest_data))
|
||||
nonce = self.cipher.extract_iv(manifest_data) + manifest_blocks
|
||||
self.cipher.set_iv(nonce)
|
||||
self.nonce_manager = NonceManager(self.repository, nonce)
|
||||
|
||||
|
||||
class FlexiKey:
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
from binascii import unhexlify
|
||||
|
||||
from ..helpers import get_security_dir
|
||||
from ..helpers import bin_to_hex
|
||||
from ..platform import SaveFile
|
||||
from ..remote import InvalidRPCMethod
|
||||
|
||||
from .low_level import bytes_to_long, long_to_bytes
|
||||
|
||||
MAX_REPRESENTABLE_NONCE = 2**64 - 1
|
||||
NONCE_SPACE_RESERVATION = 2**28 # This in units of AES blocksize (16 bytes)
|
||||
|
||||
|
||||
class NonceManager:
|
||||
def __init__(self, repository, manifest_nonce):
|
||||
self.repository = repository
|
||||
self.end_of_nonce_reservation = None
|
||||
self.manifest_nonce = manifest_nonce
|
||||
self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), "nonce")
|
||||
|
||||
def get_local_free_nonce(self):
|
||||
try:
|
||||
with open(self.nonce_file) as fd:
|
||||
return bytes_to_long(unhexlify(fd.read()))
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
def commit_local_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
if self.get_local_free_nonce() != start_nonce:
|
||||
raise Exception("nonce space reservation with mismatched previous state")
|
||||
with SaveFile(self.nonce_file, binary=False) as fd:
|
||||
fd.write(bin_to_hex(long_to_bytes(next_unreserved)))
|
||||
|
||||
def get_repo_free_nonce(self):
|
||||
try:
|
||||
return self.repository.get_free_nonce()
|
||||
except InvalidRPCMethod:
|
||||
# old server version, suppress further calls
|
||||
sys.stderr.write("Please upgrade to borg version 1.1+ on the server for safer AES-CTR nonce handling.\n")
|
||||
self.get_repo_free_nonce = lambda: None
|
||||
self.commit_repo_nonce_reservation = lambda next_unreserved, start_nonce: None
|
||||
return None
|
||||
|
||||
def commit_repo_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
self.repository.commit_nonce_reservation(next_unreserved, start_nonce)
|
||||
|
||||
def ensure_reservation(self, nonce, nonce_space_needed):
|
||||
"""
|
||||
Call this before doing encryption, give current, yet unused, integer IV as <nonce>
|
||||
and the amount of subsequent (counter-like) IVs needed as <nonce_space_needed>.
|
||||
Return value is the IV (counter) integer you shall use for encryption.
|
||||
|
||||
Note: this method may return the <nonce> you gave, if a reservation for it exists or
|
||||
can be established, so make sure you give a unused nonce.
|
||||
"""
|
||||
# Nonces may never repeat, even if a transaction aborts or the system crashes.
|
||||
# Therefore a part of the nonce space is reserved before any nonce is used for encryption.
|
||||
# As these reservations are committed to permanent storage before any nonce is used, this protects
|
||||
# against nonce reuse in crashes and transaction aborts. In that case the reservation still
|
||||
# persists and the whole reserved space is never reused.
|
||||
#
|
||||
# Local storage on the client is used to protect against an attacker that is able to rollback the
|
||||
# state of the server or can do arbitrary modifications to the repository.
|
||||
# Storage on the server is used for the multi client use case where a transaction on client A is
|
||||
# aborted and later client B writes to the repository.
|
||||
#
|
||||
# This scheme does not protect against attacker who is able to rollback the state of the server
|
||||
# or can do arbitrary modifications to the repository in the multi client usecase.
|
||||
|
||||
if self.end_of_nonce_reservation:
|
||||
# we already got a reservation, if nonce_space_needed still fits everything is ok
|
||||
next_nonce = nonce
|
||||
assert next_nonce <= self.end_of_nonce_reservation
|
||||
if next_nonce + nonce_space_needed <= self.end_of_nonce_reservation:
|
||||
return next_nonce
|
||||
|
||||
repo_free_nonce = self.get_repo_free_nonce()
|
||||
local_free_nonce = self.get_local_free_nonce()
|
||||
free_nonce_space = max(
|
||||
x
|
||||
for x in (repo_free_nonce, local_free_nonce, self.manifest_nonce, self.end_of_nonce_reservation)
|
||||
if x is not None
|
||||
)
|
||||
reservation_end = free_nonce_space + nonce_space_needed + NONCE_SPACE_RESERVATION
|
||||
assert reservation_end < MAX_REPRESENTABLE_NONCE
|
||||
self.commit_repo_nonce_reservation(reservation_end, repo_free_nonce)
|
||||
self.commit_local_nonce_reservation(reservation_end, local_free_nonce)
|
||||
self.end_of_nonce_reservation = reservation_end
|
||||
return free_nonce_space
|
||||
|
|
@ -134,8 +134,6 @@ compatMap = {
|
|||
"negotiate": ("client_data",),
|
||||
"open": ("path", "create", "lock_wait", "lock", "exclusive", "append_only"),
|
||||
"info": (),
|
||||
"get_free_nonce": (),
|
||||
"commit_nonce_reservation": ("next_unreserved", "start_nonce"),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -159,8 +157,6 @@ class RepositoryServer: # pragma: no cover
|
|||
"save_key",
|
||||
"load_key",
|
||||
"break_lock",
|
||||
"get_free_nonce",
|
||||
"commit_nonce_reservation",
|
||||
"inject_exception",
|
||||
)
|
||||
|
||||
|
|
@ -1024,14 +1020,6 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
|
|||
def load_key(self):
|
||||
"""actual remoting is done via self.call in the @api decorator"""
|
||||
|
||||
@api(since=parse_version("1.0.0"))
|
||||
def get_free_nonce(self):
|
||||
"""actual remoting is done via self.call in the @api decorator"""
|
||||
|
||||
@api(since=parse_version("1.0.0"))
|
||||
def commit_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
"""actual remoting is done via self.call in the @api decorator"""
|
||||
|
||||
@api(since=parse_version("1.0.0"))
|
||||
def break_lock(self):
|
||||
"""actual remoting is done via self.call in the @api decorator"""
|
||||
|
|
|
|||
|
|
@ -369,36 +369,6 @@ class Repository:
|
|||
# note: if we return an empty string, it means there is no repo key
|
||||
return keydata.encode("utf-8") # remote repo: msgpack issue #99, returning bytes
|
||||
|
||||
def get_free_nonce(self):
|
||||
if self.do_lock and not self.lock.got_exclusive_lock():
|
||||
raise AssertionError("bug in code, exclusive lock should exist here")
|
||||
|
||||
nonce_path = os.path.join(self.path, "nonce")
|
||||
try:
|
||||
with open(nonce_path) as fd:
|
||||
return int.from_bytes(unhexlify(fd.read()), byteorder="big")
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
def commit_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
if self.do_lock and not self.lock.got_exclusive_lock():
|
||||
raise AssertionError("bug in code, exclusive lock should exist here")
|
||||
|
||||
if self.get_free_nonce() != start_nonce:
|
||||
raise Exception("nonce space reservation with mismatched previous state")
|
||||
nonce_path = os.path.join(self.path, "nonce")
|
||||
try:
|
||||
with SaveFile(nonce_path, binary=False) as fd:
|
||||
fd.write(bin_to_hex(next_unreserved.to_bytes(8, byteorder="big")))
|
||||
except PermissionError as e:
|
||||
# error is only a problem if we even had a lock
|
||||
if self.do_lock:
|
||||
raise
|
||||
logger.warning(
|
||||
"%s: Failed writing to '%s'. This is expected when working on "
|
||||
"read-only repositories." % (e.strerror, e.filename)
|
||||
)
|
||||
|
||||
def destroy(self):
|
||||
"""Destroy the repository at `self.path`"""
|
||||
if self.append_only:
|
||||
|
|
|
|||
|
|
@ -116,12 +116,6 @@ class TestKey:
|
|||
id = bytes(32)
|
||||
id_str = bin_to_hex(id)
|
||||
|
||||
def get_free_nonce(self):
|
||||
return None
|
||||
|
||||
def commit_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
pass
|
||||
|
||||
def save_key(self, data):
|
||||
self.key_data = data
|
||||
|
||||
|
|
|
|||
|
|
@ -1,197 +0,0 @@
|
|||
import os.path
|
||||
|
||||
import pytest
|
||||
|
||||
from ..crypto import nonces
|
||||
from ..crypto.nonces import NonceManager
|
||||
from ..crypto.key import bin_to_hex
|
||||
from ..helpers import get_security_dir
|
||||
from ..remote import InvalidRPCMethod
|
||||
|
||||
|
||||
class TestNonceManager:
|
||||
class MockRepository:
|
||||
class _Location:
|
||||
orig = "/some/place"
|
||||
|
||||
_location = _Location()
|
||||
id = bytes(32)
|
||||
id_str = bin_to_hex(id)
|
||||
|
||||
def get_free_nonce(self):
|
||||
return self.next_free
|
||||
|
||||
def commit_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
assert start_nonce == self.next_free
|
||||
self.next_free = next_unreserved
|
||||
|
||||
class MockOldRepository(MockRepository):
|
||||
def get_free_nonce(self):
|
||||
raise InvalidRPCMethod("")
|
||||
|
||||
def commit_nonce_reservation(self, next_unreserved, start_nonce):
|
||||
pytest.fail("commit_nonce_reservation should never be called on an old repository")
|
||||
|
||||
def setUp(self):
|
||||
self.repository = None
|
||||
|
||||
def cache_nonce(self):
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce")) as fd:
|
||||
return fd.read()
|
||||
|
||||
def set_cache_nonce(self, nonce):
|
||||
with open(os.path.join(get_security_dir(self.repository.id_str), "nonce"), "w") as fd:
|
||||
assert fd.write(nonce)
|
||||
|
||||
def test_empty_cache_and_old_server(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockOldRepository()
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_empty_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_empty_nonce(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = None
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
# enough space in reservation
|
||||
next_nonce = manager.ensure_reservation(0x2013, 13)
|
||||
assert next_nonce == 0x2013
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
# just barely enough space in reservation
|
||||
next_nonce = manager.ensure_reservation(0x2020, 19)
|
||||
assert next_nonce == 0x2020
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
# no space in reservation
|
||||
next_nonce = manager.ensure_reservation(0x2033, 16)
|
||||
assert next_nonce == 0x2033
|
||||
assert self.cache_nonce() == "0000000000002063"
|
||||
assert self.repository.next_free == 0x2063
|
||||
|
||||
# spans reservation boundary
|
||||
next_nonce = manager.ensure_reservation(0x2043, 64)
|
||||
assert next_nonce == 0x2063
|
||||
assert self.cache_nonce() == "00000000000020c3"
|
||||
assert self.repository.next_free == 0x20C3
|
||||
|
||||
def test_sync_nonce(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
self.set_cache_nonce("0000000000002000")
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_server_just_upgraded(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = None
|
||||
self.set_cache_nonce("0000000000002000")
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_transaction_abort_no_cache(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x1000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_transaction_abort_old_server(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockOldRepository()
|
||||
self.set_cache_nonce("0000000000002000")
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x1000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
|
||||
def test_transaction_abort_on_other_client(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
self.set_cache_nonce("0000000000001000")
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x1000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
def test_interleaved(self, monkeypatch):
|
||||
monkeypatch.setattr(nonces, "NONCE_SPACE_RESERVATION", 0x20)
|
||||
|
||||
self.repository = self.MockRepository()
|
||||
self.repository.next_free = 0x2000
|
||||
self.set_cache_nonce("0000000000002000")
|
||||
|
||||
manager = NonceManager(self.repository, 0x2000)
|
||||
next_nonce = manager.ensure_reservation(0x2000, 19)
|
||||
assert next_nonce == 0x2000
|
||||
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x2033
|
||||
|
||||
# somehow the clients unlocks, another client reserves and this client relocks
|
||||
self.repository.next_free = 0x4000
|
||||
|
||||
# enough space in reservation
|
||||
next_nonce = manager.ensure_reservation(0x2013, 12)
|
||||
assert next_nonce == 0x2013
|
||||
assert self.cache_nonce() == "0000000000002033"
|
||||
assert self.repository.next_free == 0x4000
|
||||
|
||||
# spans reservation boundary
|
||||
next_nonce = manager.ensure_reservation(0x201F, 21)
|
||||
assert next_nonce == 0x4000
|
||||
assert self.cache_nonce() == "0000000000004035"
|
||||
assert self.repository.next_free == 0x4035
|
||||
|
|
@ -613,48 +613,6 @@ class QuotaTestCase(RepositoryTestCaseBase):
|
|||
assert self.repository.storage_quota_use == len(ch1) + 41 + 8 # now we have compacted.
|
||||
|
||||
|
||||
class NonceReservation(RepositoryTestCaseBase):
|
||||
def test_get_free_nonce_asserts(self):
|
||||
self.reopen(exclusive=False)
|
||||
with pytest.raises(AssertionError):
|
||||
with self.repository:
|
||||
self.repository.get_free_nonce()
|
||||
|
||||
def test_get_free_nonce(self):
|
||||
with self.repository:
|
||||
assert self.repository.get_free_nonce() is None
|
||||
|
||||
with open(os.path.join(self.repository.path, "nonce"), "w") as fd:
|
||||
fd.write("0000000000000000")
|
||||
assert self.repository.get_free_nonce() == 0
|
||||
|
||||
with open(os.path.join(self.repository.path, "nonce"), "w") as fd:
|
||||
fd.write("5000000000000000")
|
||||
assert self.repository.get_free_nonce() == 0x5000000000000000
|
||||
|
||||
def test_commit_nonce_reservation_asserts(self):
|
||||
self.reopen(exclusive=False)
|
||||
with pytest.raises(AssertionError):
|
||||
with self.repository:
|
||||
self.repository.commit_nonce_reservation(0x200, 0x100)
|
||||
|
||||
def test_commit_nonce_reservation(self):
|
||||
with self.repository:
|
||||
with pytest.raises(Exception):
|
||||
self.repository.commit_nonce_reservation(0x200, 15)
|
||||
|
||||
self.repository.commit_nonce_reservation(0x200, None)
|
||||
with open(os.path.join(self.repository.path, "nonce")) as fd:
|
||||
assert fd.read() == "0000000000000200"
|
||||
|
||||
with pytest.raises(Exception):
|
||||
self.repository.commit_nonce_reservation(0x200, 15)
|
||||
|
||||
self.repository.commit_nonce_reservation(0x400, 0x200)
|
||||
with open(os.path.join(self.repository.path, "nonce")) as fd:
|
||||
assert fd.read() == "0000000000000400"
|
||||
|
||||
|
||||
class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue