mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

api: enable giving already compressed data 

the api already offered getting compressed data,
but not giving compressed data.

thus: cache.add_chunk and key.encrypt improved.
This commit is contained in:
Thomas Waldmann 2022-05-03 04:15:01 +02:00
parent 19ad926c70
commit 0e53dc040a
2 changed files with 25 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/borg/cache.py
View File

@ -939,14 +939,17 @@ class LocalCache(CacheStatsMixin):
self.cache_config.ignored_features.update(repo_features - my_features) self.cache_config.ignored_features.update(repo_features - my_features)
self.cache_config.mandatory_features.update(repo_features & my_features) self.cache_config.mandatory_features.update(repo_features & my_features)
def add_chunk(self, id, chunk, stats, overwrite=False, wait=True): def add_chunk(self, id, chunk, stats, *, overwrite=False, wait=True, compress=True, size=None):
if not self.txn_active: if not self.txn_active:
self.begin_txn() self.begin_txn()
size = len(chunk) if size is None and compress:
size = len(chunk) # chunk is still uncompressed
refcount = self.seen_chunk(id, size) refcount = self.seen_chunk(id, size)
if refcount and not overwrite: if refcount and not overwrite:
return self.chunk_incref(id, stats) return self.chunk_incref(id, stats)
data = self.key.encrypt(id, chunk) if size is None:
raise ValueError("when giving compressed data for a new chunk, the uncompressed size must be given also")
data = self.key.encrypt(id, chunk, compress=compress)
csize = len(data) csize = len(data)
self.repository.put(id, data, wait=wait) self.repository.put(id, data, wait=wait)
self.chunks.add(id, 1, size, csize) self.chunks.add(id, 1, size, csize)
@ -1103,15 +1106,18 @@ Chunk index: {0.total_unique_chunks:20d} unknown"""
def memorize_file(self, hashed_path, path_hash, st, ids): def memorize_file(self, hashed_path, path_hash, st, ids):
pass pass
def add_chunk(self, id, chunk, stats, overwrite=False, wait=True): def add_chunk(self, id, chunk, stats, *, overwrite=False, wait=True, compress=True, size=None):
assert not overwrite, 'AdHocCache does not permit overwrites — trying to use it for recreate?' assert not overwrite, 'AdHocCache does not permit overwrites — trying to use it for recreate?'
if not self._txn_active: if not self._txn_active:
self.begin_txn() self.begin_txn()
size = len(chunk) if size is None and compress:
size = len(chunk) # chunk is still uncompressed
if size is None:
raise ValueError("when giving compressed data for a chunk, the uncompressed size must be given also")
refcount = self.seen_chunk(id, size) refcount = self.seen_chunk(id, size)
if refcount: if refcount:
return self.chunk_incref(id, stats, size=size) return self.chunk_incref(id, stats, size=size)
data = self.key.encrypt(id, chunk) data = self.key.encrypt(id, chunk, compress=compress)
csize = len(data) csize = len(data)
self.repository.put(id, data, wait=wait) self.repository.put(id, data, wait=wait)
self.chunks.add(id, 1, size, csize) self.chunks.add(id, 1, size, csize)

22
src/borg/crypto/key.py
View File

@ -167,7 +167,7 @@ class KeyBase:
""" """
raise NotImplementedError raise NotImplementedError
def encrypt(self, id, data): def encrypt(self, id, data, compress=True):
pass pass
def decrypt(self, id, data, decompress=True): def decrypt(self, id, data, decompress=True):
@ -273,8 +273,9 @@ class PlaintextKey(KeyBase):
def id_hash(self, data): def id_hash(self, data):
return sha256(data).digest() return sha256(data).digest()
def encrypt(self, id, data): def encrypt(self, id, data, compress=True):
data = self.compressor.compress(data) if compress:
data = self.compressor.compress(data)
return b''.join([self.TYPE_STR, data]) return b''.join([self.TYPE_STR, data])
def decrypt(self, id, data, decompress=True): def decrypt(self, id, data, decompress=True):
@ -349,8 +350,9 @@ class AESKeyBase(KeyBase):
logically_encrypted = True logically_encrypted = True
def encrypt(self, id, data): def encrypt(self, id, data, compress=True):
data = self.compressor.compress(data) if compress:
data = self.compressor.compress(data)
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(), next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(),
self.cipher.block_count(len(data))) self.cipher.block_count(len(data)))
return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv) return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)
@ -809,8 +811,9 @@ class AuthenticatedKeyBase(AESKeyBase, FlexiKey):
if manifest_data is not None: if manifest_data is not None:
self.assert_type(manifest_data[0]) self.assert_type(manifest_data[0])
def encrypt(self, id, data): def encrypt(self, id, data, compress=True):
data = self.compressor.compress(data) if compress:
data = self.compressor.compress(data)
return b''.join([self.TYPE_STR, data]) return b''.join([self.TYPE_STR, data])
def decrypt(self, id, data, decompress=True): def decrypt(self, id, data, decompress=True):
@ -865,9 +868,10 @@ class AEADKeyBase(KeyBase):
MAX_IV = 2 ** 48 - 1 MAX_IV = 2 ** 48 - 1
def encrypt(self, id, data): def encrypt(self, id, data, compress=True):
# to encrypt new data in this session we use always self.cipher and self.sessionid # to encrypt new data in this session we use always self.cipher and self.sessionid
data = self.compressor.compress(data) if compress:
data = self.compressor.compress(data)
reserved = b'\0' reserved = b'\0'
iv = self.cipher.next_iv() iv = self.cipher.next_iv()
if iv > self.MAX_IV: # see the data-structures docs about why the IV range is enough if iv > self.MAX_IV: # see the data-structures docs about why the IV range is enough