mirror of https://github.com/borgbackup/borg.git
simplify: iter_objects always returns (..., size, data)
data might be None (if read_data is False). also removed the include_data argument, not needed any more.
This commit is contained in:
parent
e827f98c45
commit
b64427c480
|
@ -830,7 +830,7 @@ class Repository:
|
||||||
freeable_ratio * 100.0,
|
freeable_ratio * 100.0,
|
||||||
freeable_space,
|
freeable_space,
|
||||||
)
|
)
|
||||||
for tag, key, offset, data in self.io.iter_objects(segment, include_data=True):
|
for tag, key, offset, _, data in self.io.iter_objects(segment):
|
||||||
if tag == TAG_COMMIT:
|
if tag == TAG_COMMIT:
|
||||||
continue
|
continue
|
||||||
in_index = self.index.get(key)
|
in_index = self.index.get(key)
|
||||||
|
@ -961,7 +961,7 @@ class Repository:
|
||||||
def _update_index(self, segment, objects, report=None):
|
def _update_index(self, segment, objects, report=None):
|
||||||
"""some code shared between replay_segments and check"""
|
"""some code shared between replay_segments and check"""
|
||||||
self.segments[segment] = 0
|
self.segments[segment] = 0
|
||||||
for tag, key, offset, size in objects:
|
for tag, key, offset, size, _ in objects:
|
||||||
if tag in (TAG_PUT2, TAG_PUT):
|
if tag in (TAG_PUT2, TAG_PUT):
|
||||||
try:
|
try:
|
||||||
# If this PUT supersedes an older PUT, mark the old segment for compaction and count the free space
|
# If this PUT supersedes an older PUT, mark the old segment for compaction and count the free space
|
||||||
|
@ -1011,7 +1011,7 @@ class Repository:
|
||||||
return
|
return
|
||||||
|
|
||||||
self.compact[segment] = 0
|
self.compact[segment] = 0
|
||||||
for tag, key, offset, size in self.io.iter_objects(segment, read_data=False):
|
for tag, key, offset, size, _ in self.io.iter_objects(segment, read_data=False):
|
||||||
if tag in (TAG_PUT2, TAG_PUT):
|
if tag in (TAG_PUT2, TAG_PUT):
|
||||||
in_index = self.index.get(key)
|
in_index = self.index.get(key)
|
||||||
if not in_index or (in_index.segment, in_index.offset) != (segment, offset):
|
if not in_index or (in_index.segment, in_index.offset) != (segment, offset):
|
||||||
|
@ -1165,8 +1165,8 @@ class Repository:
|
||||||
if segment is not None and current_segment > segment:
|
if segment is not None and current_segment > segment:
|
||||||
break
|
break
|
||||||
try:
|
try:
|
||||||
for tag, key, current_offset, data in self.io.iter_objects(
|
for tag, key, current_offset, _, data in self.io.iter_objects(
|
||||||
segment=current_segment, offset=offset or 0, include_data=True
|
segment=current_segment, offset=offset or 0
|
||||||
):
|
):
|
||||||
if offset is not None and current_offset > offset:
|
if offset is not None and current_offset > offset:
|
||||||
break
|
break
|
||||||
|
@ -1229,10 +1229,10 @@ class Repository:
|
||||||
start_segment, start_offset, _ = (0, 0, 0) if at_start else self.index[marker]
|
start_segment, start_offset, _ = (0, 0, 0) if at_start else self.index[marker]
|
||||||
result = []
|
result = []
|
||||||
for segment, filename in self.io.segment_iterator(start_segment):
|
for segment, filename in self.io.segment_iterator(start_segment):
|
||||||
obj_iterator = self.io.iter_objects(segment, start_offset, read_data=False, include_data=False)
|
obj_iterator = self.io.iter_objects(segment, start_offset, read_data=False)
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
tag, id, offset, size = next(obj_iterator)
|
tag, id, offset, size, _ = next(obj_iterator)
|
||||||
except (StopIteration, IntegrityError):
|
except (StopIteration, IntegrityError):
|
||||||
# either end-of-segment or an error - we can not seek to objects at
|
# either end-of-segment or an error - we can not seek to objects at
|
||||||
# higher offsets than one that has an error in the header fields.
|
# higher offsets than one that has an error in the header fields.
|
||||||
|
@ -1458,7 +1458,7 @@ class LoggedIO:
|
||||||
seen_commit = False
|
seen_commit = False
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
tag, key, offset, _ = next(iterator)
|
tag, key, offset, _, _ = next(iterator)
|
||||||
except IntegrityError:
|
except IntegrityError:
|
||||||
return False
|
return False
|
||||||
except StopIteration:
|
except StopIteration:
|
||||||
|
@ -1560,15 +1560,13 @@ class LoggedIO:
|
||||||
fd.seek(0)
|
fd.seek(0)
|
||||||
return fd.read(MAGIC_LEN)
|
return fd.read(MAGIC_LEN)
|
||||||
|
|
||||||
def iter_objects(self, segment, offset=0, include_data=False, read_data=True):
|
def iter_objects(self, segment, offset=0, read_data=True):
|
||||||
"""
|
"""
|
||||||
Return object iterator for *segment*.
|
Return object iterator for *segment*.
|
||||||
|
|
||||||
If read_data is False then include_data must be False as well.
|
|
||||||
|
|
||||||
See the _read() docstring about confidence in the returned data.
|
See the _read() docstring about confidence in the returned data.
|
||||||
|
|
||||||
The iterator returns four-tuples of (tag, key, offset, data|size).
|
The iterator returns five-tuples of (tag, key, offset, size, data).
|
||||||
"""
|
"""
|
||||||
fd = self.get_fd(segment)
|
fd = self.get_fd(segment)
|
||||||
fd.seek(offset)
|
fd.seek(offset)
|
||||||
|
@ -1584,10 +1582,9 @@ class LoggedIO:
|
||||||
size, tag, key, data = self._read(
|
size, tag, key, data = self._read(
|
||||||
fd, header, segment, offset, (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), read_data=read_data
|
fd, header, segment, offset, (TAG_PUT2, TAG_DELETE, TAG_COMMIT, TAG_PUT), read_data=read_data
|
||||||
)
|
)
|
||||||
if include_data:
|
# tuple[3]: corresponds to len(data) == length of the full chunk payload (meta_len+enc_meta+enc_data)
|
||||||
yield tag, key, offset, data
|
# tuple[4]: data will be None if read_data is False.
|
||||||
else:
|
yield tag, key, offset, size - header_size(tag), data
|
||||||
yield tag, key, offset, size - header_size(tag) # corresponds to len(data)
|
|
||||||
assert size >= 0
|
assert size >= 0
|
||||||
offset += size
|
offset += size
|
||||||
# we must get the fd via get_fd() here again as we yielded to our caller and it might
|
# we must get the fd via get_fd() here again as we yielded to our caller and it might
|
||||||
|
|
|
@ -60,7 +60,7 @@ class RepositoryTestCaseBase(BaseTestCase):
|
||||||
H_trans[None] = -1 # key == None appears in commits
|
H_trans[None] = -1 # key == None appears in commits
|
||||||
tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"}
|
tag_trans = {TAG_PUT2: "put2", TAG_PUT: "put", TAG_DELETE: "del", TAG_COMMIT: "comm"}
|
||||||
for segment, fn in self.repository.io.segment_iterator():
|
for segment, fn in self.repository.io.segment_iterator():
|
||||||
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
for tag, key, offset, size, _ in self.repository.io.iter_objects(segment):
|
||||||
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
|
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
@ -372,7 +372,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||||
self.repo_dump("d1 cc")
|
self.repo_dump("d1 cc")
|
||||||
last_segment = self.repository.io.get_latest_segment() - 1
|
last_segment = self.repository.io.get_latest_segment() - 1
|
||||||
num_deletes = 0
|
num_deletes = 0
|
||||||
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
|
for tag, key, offset, size, _ in self.repository.io.iter_objects(last_segment):
|
||||||
if tag == TAG_DELETE:
|
if tag == TAG_DELETE:
|
||||||
assert key == H(1)
|
assert key == H(1)
|
||||||
num_deletes += 1
|
num_deletes += 1
|
||||||
|
@ -384,7 +384,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
||||||
assert last_segment not in self.repository.compact
|
assert last_segment not in self.repository.compact
|
||||||
assert not self.repository.io.segment_exists(last_segment)
|
assert not self.repository.io.segment_exists(last_segment)
|
||||||
for segment, _ in self.repository.io.segment_iterator():
|
for segment, _ in self.repository.io.segment_iterator():
|
||||||
for tag, key, offset, size in self.repository.io.iter_objects(segment):
|
for tag, key, offset, size, _ in self.repository.io.iter_objects(segment):
|
||||||
assert tag != TAG_DELETE
|
assert tag != TAG_DELETE
|
||||||
assert key != H(1)
|
assert key != H(1)
|
||||||
# after compaction, there should be no empty shadowed_segments lists left over.
|
# after compaction, there should be no empty shadowed_segments lists left over.
|
||||||
|
|
Loading…
Reference in New Issue