mirror of https://github.com/borgbackup/borg.git
Merge pull request #5614 from ThomasWaldmann/remove-empty-shadowed-segments-list-master
remove empty shadowed_segments lists, fixes #5275
This commit is contained in:
commit
e3d8b7c9ea
|
@ -794,6 +794,8 @@ class Repository:
|
||||||
try:
|
try:
|
||||||
self.shadow_index[key].remove(segment)
|
self.shadow_index[key].remove(segment)
|
||||||
except (KeyError, ValueError):
|
except (KeyError, ValueError):
|
||||||
|
# do not remove entry with empty shadowed_segments list here,
|
||||||
|
# it is needed for shadowed_put_exists code (see below)!
|
||||||
pass
|
pass
|
||||||
elif tag == TAG_DELETE and not in_index:
|
elif tag == TAG_DELETE and not in_index:
|
||||||
# If the shadow index doesn't contain this key, then we can't say if there's a shadowed older tag,
|
# If the shadow index doesn't contain this key, then we can't say if there's a shadowed older tag,
|
||||||
|
@ -845,6 +847,11 @@ class Repository:
|
||||||
new_segment, size = self.io.write_delete(key)
|
new_segment, size = self.io.write_delete(key)
|
||||||
self.compact[new_segment] += size
|
self.compact[new_segment] += size
|
||||||
segments.setdefault(new_segment, 0)
|
segments.setdefault(new_segment, 0)
|
||||||
|
else:
|
||||||
|
# we did not keep the delete tag for key (see if-branch)
|
||||||
|
if not self.shadow_index[key]:
|
||||||
|
# shadowed segments list is empty -> remove it
|
||||||
|
del self.shadow_index[key]
|
||||||
assert segments[segment] == 0, 'Corrupted segment reference count - corrupted index or hints'
|
assert segments[segment] == 0, 'Corrupted segment reference count - corrupted index or hints'
|
||||||
unused.append(segment)
|
unused.append(segment)
|
||||||
pi.show()
|
pi.show()
|
||||||
|
|
|
@ -14,7 +14,7 @@ from ..helpers import IntegrityError
|
||||||
from ..helpers import msgpack
|
from ..helpers import msgpack
|
||||||
from ..locking import Lock, LockFailed
|
from ..locking import Lock, LockFailed
|
||||||
from ..remote import RemoteRepository, InvalidRPCMethod, PathNotAllowed, ConnectionClosedWithHint, handle_remote_line
|
from ..remote import RemoteRepository, InvalidRPCMethod, PathNotAllowed, ConnectionClosedWithHint, handle_remote_line
|
||||||
from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE
|
from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT, TAG_COMMIT
|
||||||
from . import BaseTestCase
|
from . import BaseTestCase
|
||||||
from .hashindex import H
|
from .hashindex import H
|
||||||
|
|
||||||
|
@ -54,6 +54,16 @@ class RepositoryTestCaseBase(BaseTestCase):
|
||||||
self.repository.put(H(2), b'boo')
|
self.repository.put(H(2), b'boo')
|
||||||
self.repository.delete(H(3))
|
self.repository.delete(H(3))
|
||||||
|
|
||||||
|
def repo_dump(self, label=None):
|
||||||
|
label = label + ': ' if label is not None else ''
|
||||||
|
H_trans = {H(i): i for i in range(10)}
|
||||||
|
H_trans[None] = -1 # key == None appears in commits
|
||||||
|
tag_trans = {TAG_PUT: 'put', TAG_DELETE: 'del', TAG_COMMIT: 'comm'}
|
||||||
|
for segment, fn in self.repository.io.segment_iterator():
|
||||||
|
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
||||||
|
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
class RepositoryTestCase(RepositoryTestCaseBase):
|
class RepositoryTestCase(RepositoryTestCaseBase):
|
||||||
|
|
||||||
|
@ -315,8 +325,10 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||||
self.repository.put(H(1), b'1')
|
self.repository.put(H(1), b'1')
|
||||||
self.repository.put(H(2), b'2')
|
self.repository.put(H(2), b'2')
|
||||||
self.repository.commit(compact=False)
|
self.repository.commit(compact=False)
|
||||||
|
self.repo_dump('p1 p2 c')
|
||||||
self.repository.delete(H(1))
|
self.repository.delete(H(1))
|
||||||
self.repository.commit(compact=True)
|
self.repository.commit(compact=True)
|
||||||
|
self.repo_dump('d1 cc')
|
||||||
last_segment = self.repository.io.get_latest_segment() - 1
|
last_segment = self.repository.io.get_latest_segment() - 1
|
||||||
num_deletes = 0
|
num_deletes = 0
|
||||||
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
|
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
|
||||||
|
@ -327,11 +339,16 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||||
assert last_segment in self.repository.compact
|
assert last_segment in self.repository.compact
|
||||||
self.repository.put(H(3), b'3')
|
self.repository.put(H(3), b'3')
|
||||||
self.repository.commit(compact=True)
|
self.repository.commit(compact=True)
|
||||||
|
self.repo_dump('p3 cc')
|
||||||
assert last_segment not in self.repository.compact
|
assert last_segment not in self.repository.compact
|
||||||
assert not self.repository.io.segment_exists(last_segment)
|
assert not self.repository.io.segment_exists(last_segment)
|
||||||
for segment, _ in self.repository.io.segment_iterator():
|
for segment, _ in self.repository.io.segment_iterator():
|
||||||
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
||||||
assert tag != TAG_DELETE
|
assert tag != TAG_DELETE
|
||||||
|
assert key != H(1)
|
||||||
|
# after compaction, there should be no empty shadowed_segments lists left over.
|
||||||
|
# we have no put or del any more for H(1), so we lost knowledge about H(1).
|
||||||
|
assert H(1) not in self.repository.shadow_index
|
||||||
|
|
||||||
def test_shadowed_entries_are_preserved(self):
|
def test_shadowed_entries_are_preserved(self):
|
||||||
get_latest_segment = self.repository.io.get_latest_segment
|
get_latest_segment = self.repository.io.get_latest_segment
|
||||||
|
@ -372,16 +389,19 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||||
self.repository.delete(H(1))
|
self.repository.delete(H(1))
|
||||||
assert self.repository.shadow_index[H(1)] == [0]
|
assert self.repository.shadow_index[H(1)] == [0]
|
||||||
self.repository.commit(compact=True)
|
self.repository.commit(compact=True)
|
||||||
|
self.repo_dump('p1 d1 cc')
|
||||||
# note how an empty list means that nothing is shadowed for sure
|
# note how an empty list means that nothing is shadowed for sure
|
||||||
assert self.repository.shadow_index[H(1)] == []
|
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
|
||||||
self.repository.put(H(1), b'1')
|
self.repository.put(H(1), b'1')
|
||||||
self.repository.delete(H(1))
|
self.repository.delete(H(1))
|
||||||
|
self.repo_dump('p1 d1')
|
||||||
# 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
|
# 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
|
||||||
assert self.repository.shadow_index[H(1)] == [4]
|
assert self.repository.shadow_index[H(1)] == [4]
|
||||||
self.repository.rollback()
|
self.repository.rollback()
|
||||||
|
self.repo_dump('r')
|
||||||
self.repository.put(H(2), b'1')
|
self.repository.put(H(2), b'1')
|
||||||
# After the rollback segment 4 shouldn't be considered anymore
|
# After the rollback segment 4 shouldn't be considered anymore
|
||||||
assert self.repository.shadow_index[H(1)] == []
|
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
|
||||||
|
|
||||||
|
|
||||||
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
|
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
|
||||||
|
@ -826,7 +846,7 @@ class RepositoryHintsTestCase(RepositoryTestCaseBase):
|
||||||
self.repository.put(H(42), b'foobar') # see also do_compact()
|
self.repository.put(H(42), b'foobar') # see also do_compact()
|
||||||
self.repository.commit(compact=True, threshold=0.0) # compact completely!
|
self.repository.commit(compact=True, threshold=0.0) # compact completely!
|
||||||
# nothing to compact any more! no info left about stuff that does not exist any more:
|
# nothing to compact any more! no info left about stuff that does not exist any more:
|
||||||
self.assert_equal(self.repository.shadow_index[H(0)], [])
|
self.assert_not_in(H(0), self.repository.shadow_index)
|
||||||
# segment 0 was compacted away, no info about it left:
|
# segment 0 was compacted away, no info about it left:
|
||||||
self.assert_not_in(0, self.repository.compact)
|
self.assert_not_in(0, self.repository.compact)
|
||||||
self.assert_not_in(0, self.repository.segments)
|
self.assert_not_in(0, self.repository.segments)
|
||||||
|
|
Loading…
Reference in New Issue