mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

simplify: iter_objects always returns (..., size, data) 

data might be None (if read_data is False).

also removed the include_data argument, not needed any more.
This commit is contained in:
Thomas Waldmann 2022-09-08 12:38:55 +02:00
parent e827f98c45
commit b64427c480
2 changed files with 16 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/borg/repository.py
View File

@ -830,7 +830,7 @@ class Repository:
freeable_ratio * 100.0, freeable_ratio * 100.0,
freeable_space, freeable_space,
) )
for tag, key, offset, data in self.io.iter_objects(segment, include_data=True): for tag, key, offset, _, data in self.io.iter_objects(segment):
if tag == TAG_COMMIT: if tag == TAG_COMMIT:
continue continue
in_index = self.index.get(key) in_index = self.index.get(key)
@ -961,7 +961,7 @@ class Repository:
def _update_index(self, segment, objects, report=None): def _update_index(self, segment, objects, report=None):
"""some code shared between replay_segments and check""" """some code shared between replay_segments and check"""
self.segments[segment] = 0 self.segments[segment] = 0
for tag, key, offset, size in objects: for tag, key, offset, size, _ in objects:
if tag in (TAG_PUT2, TAG_PUT): if tag in (TAG_PUT2, TAG_PUT):
try: try:
# If this PUT supersedes an older PUT, mark the old segment for compaction and count the free space # If this PUT supersedes an older PUT, mark the old segment for compaction and count the free space
@ -1011,7 +1011,7 @@ class Repository:
return return
self.compact[segment] = 0 self.compact[segment] = 0
for tag, key, offset, size in self.io.iter_objects(segment, read_data=False): for tag, key, offset, size, _ in self.io.iter_objects(segment, read_data=False):
if tag in (TAG_PUT2, TAG_PUT): if tag in (TAG_PUT2, TAG_PUT):
in_index = self.index.get(key) in_index = self.index.get(key)
if not in_index or (in_index.segment, in_index.offset) != (segment, offset): if not in_index or (in_index.segment, in_index.offset) != (segment, offset):
@ -1165,8 +1165,8 @@ class Repository:
if segment is not None and current_segment > segment: if segment is not None and current_segment > segment:
break break
try: try:
for tag, key, current_offset, data in self.io.iter_objects( for tag, key, current_offset, _, data in self.io.iter_objects(
segment=current_segment, offset=offset or 0, include_data=True segment=current_segment, offset=offset or 0
): ):
if offset is not None and current_offset > offset: if offset is not None and current_offset > offset:
break break
@ -1229,10 +1229,10 @@ class Repository:
start_segment, start_offset, _ = (0, 0, 0) if at_start else self.index[marker] start_segment, start_offset, _ = (0, 0, 0) if at_start else self.index[marker]
result = [] result = []
for segment, filename in self.io.segment_iterator(start_segment): for segment, filename in self.io.segment_iterator(start_segment):
obj_iterator = self.io.iter_objects(segment, start_offset, read_data=False, include_data=False) obj_iterator = self.io.iter_objects(segment, start_offset, read_data=False)
while True: while True:
try: try:
tag, id, offset, size = next(obj_iterator) tag, id, offset, size, _ = next(obj_iterator)
except (StopIteration, IntegrityError): except (StopIteration, IntegrityError):
# either end-of-segment or an error - we can not seek to objects at # either end-of-segment or an error - we can not seek to objects at
# higher offsets than one that has an error in the header fields. # higher offsets than one that has an error in the header fields.
@ -1458,7 +1458,7 @@ class LoggedIO:
seen_commit = False seen_commit = False
while True: while True:
try: try:
tag, key, offset, _ = next(iterator) tag, key, offset, _, _ = next(iterator)
except IntegrityError: except IntegrityError:
return False return False
except StopIteration: except StopIteration:
@ -1560,15 +1560,13 @@ class LoggedIO:
fd.seek(0) fd.seek(0)
return fd.read(MAGIC_LEN) return fd.read(MAGIC_LEN)
def iter_objects(self, segment, offset=0, include_data=False, read_data=True): def iter_objects(self, segment, offset=0, read_data=True):
""" """
Return object iterator for *segment*. Return object iterator for *segment*.
If read_data is False then include_data must be False as well.
See the _read() docstring about confidence in the returned data. See the _read() docstring about confidence in the returned data.
The iterator returns four-tuples of (tag, key, offset, data|size). The iterator returns five-tuples of (tag, key, offset, size, data).
""" """
fd = self.get_fd(segment) fd = self.get_fd(segment)
fd.seek(offset) fd.seek(offset)
@ -1584,10 +1582,9 @@ class LoggedIO:
size, tag, key, data = self._read( size, tag, key, data = self._read(
fd, header, segment, offset, (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), read_data=read_data fd, header, segment, offset, (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), read_data=read_data
) )
if include_data: # tuple[3]: corresponds to len(data) == length of the full chunk payload (meta_len+enc_meta+enc_data)
yield tag, key, offset, data # tuple[4]: data will be None if read_data is False.
else: yield tag, key, offset, size - header_size(tag), data
yield tag, key, offset, size - header_size(tag) # corresponds to len(data)
assert size >= 0 assert size >= 0
offset += size offset += size
# we must get the fd via get_fd() here again as we yielded to our caller and it might # we must get the fd via get_fd() here again as we yielded to our caller and it might

6
src/borg/testsuite/repository.py
View File

@ -60,7 +60,7 @@ class RepositoryTestCaseBase(BaseTestCase):
H_trans[None] = -1 # key == None appears in commits H_trans[None] = -1 # key == None appears in commits
tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"} tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"}
for segment, fn in self.repository.io.segment_iterator(): for segment, fn in self.repository.io.segment_iterator():
for tag, key, offset, size in self.repository.io.iter_objects(segment): for tag, key, offset, size, _ in self.repository.io.iter_objects(segment):
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size)) print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
print() print()
@ -372,7 +372,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.repo_dump("d1 cc") self.repo_dump("d1 cc")
last_segment = self.repository.io.get_latest_segment() - 1 last_segment = self.repository.io.get_latest_segment() - 1
num_deletes = 0 num_deletes = 0
for tag, key, offset, size in self.repository.io.iter_objects(last_segment): for tag, key, offset, size, _ in self.repository.io.iter_objects(last_segment):
if tag == TAG_DELETE: if tag == TAG_DELETE:
assert key == H(1) assert key == H(1)
num_deletes += 1 num_deletes += 1
@ -384,7 +384,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
assert last_segment not in self.repository.compact assert last_segment not in self.repository.compact
assert not self.repository.io.segment_exists(last_segment) assert not self.repository.io.segment_exists(last_segment)
for segment, _ in self.repository.io.segment_iterator(): for segment, _ in self.repository.io.segment_iterator():
for tag, key, offset, size in self.repository.io.iter_objects(segment): for tag, key, offset, size, _ in self.repository.io.iter_objects(segment):
assert tag != TAG_DELETE assert tag != TAG_DELETE
assert key != H(1) assert key != H(1)
# after compaction, there should be no empty shadowed_segments lists left over. # after compaction, there should be no empty shadowed_segments lists left over.