mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-02-21 13:47:16 +00:00
Code Issues Releases Wiki Activity

Merge pull request #6119 from ThomasWaldmann/repo-review-master

Browse source 
repository: quota computation fixes for delayed compaction
This commit is contained in:
TW 2022-01-23 13:34:05 +01:00 committed by GitHub
commit 91e3b0cd8f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/borg/repository.py
View file

@ -806,6 +806,7 @@ def complete_xfer(intermediate=True):
# do not remove entry with empty shadowed_segments list here,
# it is needed for shadowed_put_exists code (see below)!
pass
self.storage_quota_use -= len(data) + self.io.put_header_fmt.size
elif tag == TAG_DELETE and not in_index:
# If the shadow index doesn't contain this key, then we can't say if there's a shadowed older tag,
# therefore we do not drop the delete, but write it to a current segment.
@ -906,7 +907,7 @@ def _update_index(self, segment, objects, report=None):
pass
self.index[key] = segment, offset
self.segments[segment] += 1
self.storage_quota_use += size
self.storage_quota_use += size # note: size already includes the put_header_fmt overhead
elif tag == TAG_DELETE:
try:
# if the deleted PUT is not in the index, there is nothing to clean up
@ -919,7 +920,6 @@ def _update_index(self, segment, objects, report=None):
# is already gone, then it was already compacted.
self.segments[s] -= 1
size = self.io.read(s, offset, key, read_data=False)
self.storage_quota_use -= size
self.compact[s] += size
elif tag == TAG_COMMIT:
continue
@ -930,7 +930,7 @@ def _update_index(self, segment, objects, report=None):
else:
report(msg)
if self.segments[segment] == 0:
self.compact[segment] += self.io.segment_size(segment)
self.compact[segment] = self.io.segment_size(segment)
def _rebuild_sparse(self, segment):
"""Rebuild sparse bytes count for a single segment relative to the current index."""
@ -1239,7 +1239,6 @@ def _delete(self, id, segment, offset, *, update_shadow_index):
self.shadow_index.setdefault(id, []).append(segment)
self.segments[segment] -= 1
size = self.io.read(segment, offset, id, read_data=False)
self.storage_quota_use -= size
self.compact[segment] += size
segment, size = self.io.write_delete(id)
self.compact[segment] += size

11
src/borg/testsuite/repository.py
View file

@ -463,13 +463,16 @@ def test_tracking(self):
self.repository.put(H(2), bytes(5678))
assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41
self.repository.delete(H(1))
assert self.repository.storage_quota_use == 5678 + 41
assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41 # we have not compacted yet
self.repository.commit(compact=False)
assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41 # we have not compacted yet
self.reopen()
with self.repository:
# Open new transaction; hints and thus quota data is not loaded unless needed.
self.repository.put(H(3), b'')
self.repository.delete(H(3))
assert self.repository.storage_quota_use == 1234 + 5678 + 3 * 41 # we have not compacted yet
self.repository.commit(compact=True)
assert self.repository.storage_quota_use == 5678 + 41
def test_exceed_quota(self):
@ -486,10 +489,12 @@ def test_exceed_quota(self):
assert self.repository.storage_quota_use == 82
self.reopen()
with self.repository:
self.repository.storage_quota = 50
self.repository.storage_quota = 100
# Open new transaction; hints and thus quota data is not loaded unless needed.
self.repository.put(H(1), b'')
assert self.repository.storage_quota_use == 41
assert self.repository.storage_quota_use == 82 # we have 2 puts for H(1) here and not yet compacted.
self.repository.commit(compact=True)
assert self.repository.storage_quota_use == 41 # now we have compacted.
class NonceReservation(RepositoryTestCaseBase):