1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-02-23 06:31:58 +00:00

Merge pull request #5263 from ThomasWaldmann/persist-shadow-index

persist shadow_index in between borg runs, fixes #4830
This commit is contained in:
TW 2020-07-28 22:52:58 +02:00 committed by GitHub
commit 85e6c0afd8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 1 deletions

View file

@ -160,6 +160,7 @@ def __init__(self, path, create=False, exclusive=False, lock_wait=None, lock=Tru
# This is an index of shadowed log entries during this transaction. Consider the following sequence:
# segment_n PUT A, segment_x DELETE A
# After the "DELETE A" in segment_x the shadow index will contain "A -> [n]".
# .delete() is updating this index, it is persisted into "hints" file and is later used by .compact_segments().
self.shadow_index = {}
self._active_txn = False
self.lock_wait = lock_wait
@ -570,6 +571,7 @@ def prepare_txn(self, transaction_id, do_cleanup=True):
self.segments = hints[b'segments']
self.compact = FreeSpace()
self.storage_quota_use = 0
self.shadow_index = {}
for segment in sorted(hints[b'compact']):
logger.debug('Rebuilding sparse info for segment %d', segment)
self._rebuild_sparse(segment)
@ -580,6 +582,7 @@ def prepare_txn(self, transaction_id, do_cleanup=True):
self.segments = hints[b'segments']
self.compact = FreeSpace(hints[b'compact'])
self.storage_quota_use = hints.get(b'storage_quota_use', 0)
self.shadow_index = hints.get(b'shadow_index', {})
self.log_storage_quota()
# Drop uncommitted segments in the shadow index
for key, shadowed_segments in self.shadow_index.items():
@ -600,6 +603,7 @@ def rename_tmp(file):
b'segments': self.segments,
b'compact': self.compact,
b'storage_quota_use': self.storage_quota_use,
b'shadow_index': self.shadow_index,
}
integrity = {
# Integrity version started at 2, the current hints version.
@ -975,7 +979,7 @@ def report_error(msg):
segments_transaction_id = self.io.get_segments_transaction_id()
logger.debug('Segment transaction is %s', segments_transaction_id)
logger.debug('Determined transaction is %s', transaction_id)
self.prepare_txn(None) # self.index, self.compact, self.segments all empty now!
self.prepare_txn(None) # self.index, self.compact, self.segments, self.shadow_index all empty now!
segment_count = sum(1 for _ in self.io.segment_iterator())
logger.debug('Found %d segments', segment_count)

View file

@ -792,6 +792,46 @@ def test_crash_before_compact(self):
self.assert_equal(self.repository.get(H(0)), b'data2')
class RepositoryHintsTestCase(RepositoryTestCaseBase):
def test_hints_persistence(self):
self.repository.put(H(0), b'data')
self.repository.delete(H(0))
self.repository.commit(compact=False)
shadow_index_expected = self.repository.shadow_index
compact_expected = self.repository.compact
segments_expected = self.repository.segments
# close and re-open the repository (create fresh Repository instance) to
# check whether hints were persisted to / reloaded from disk
self.reopen()
with self.repository:
# see also do_compact()
self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data
# check if hints persistence worked:
self.assert_equal(shadow_index_expected, self.repository.shadow_index)
self.assert_equal(compact_expected, self.repository.compact)
del self.repository.segments[2] # ignore the segment created by put(H(42), ...)
self.assert_equal(segments_expected, self.repository.segments)
def test_hints_behaviour(self):
self.repository.put(H(0), b'data')
self.assert_equal(self.repository.shadow_index, {})
self.assert_true(len(self.repository.compact) == 0)
self.repository.delete(H(0))
self.repository.commit(compact=False)
# now there should be an entry for H(0) in shadow_index
self.assert_in(H(0), self.repository.shadow_index)
self.assert_equal(len(self.repository.shadow_index[H(0)]), 1)
self.assert_in(0, self.repository.compact) # segment 0 can be compacted
self.repository.put(H(42), b'foobar') # see also do_compact()
self.repository.commit(compact=True, threshold=0.0) # compact completely!
# nothing to compact any more! no info left about stuff that does not exist any more:
self.assert_equal(self.repository.shadow_index[H(0)], [])
# segment 0 was compacted away, no info about it left:
self.assert_not_in(0, self.repository.compact)
self.assert_not_in(0, self.repository.segments)
class RemoteRepositoryTestCase(RepositoryTestCase):
repository = None # type: RemoteRepository