mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

rename local preload() function to not overwrite keyword argument of same name 

The locally defined preload() function overwrites the preload boolean keyword
argument, always evaluating to true, so preloading is done, even when not
requested by the caller, causing a memory leak.
Also move its definition outside of the loop.

This issue was found by Antonio Larrosa in borg issue #5202.
This commit is contained in:
Elmar Hoffmann 2020-06-01 13:22:25 +02:00
parent 9019b48fd5
commit dad3aa9dae
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/borg/archive.py
View File

@ -262,6 +262,9 @@ class DownloadPipeline:
Warning: if *preload* is True then all data chunks of every yielded item have to be retrieved,
otherwise preloaded chunks will accumulate in RemoteRepository and create a memory leak.
"""
def _preload(chunks):
self.repository.preload([c.id for c in chunks])
masters_preloaded = set()
unpacker = msgpack.Unpacker(use_list=False)
for data in self.fetch_many(ids):
@ -271,9 +274,6 @@ class DownloadPipeline:
if 'chunks' in item:
item.chunks = [ChunkListEntry(*e) for e in item.chunks]
def preload(chunks):
self.repository.preload([c.id for c in chunks])
if filter:
items = [item for item in items if filter(item)]
@ -286,7 +286,7 @@ class DownloadPipeline:
# due to a side effect of the filter() call, we now have hardlink_masters dict populated.
for item in items:
if 'chunks' in item: # regular file, maybe a hardlink master
preload(item.chunks)
_preload(item.chunks)
# if this is a hardlink master, remember that we already preloaded it:
if 'source' not in item and hardlinkable(item.mode) and item.get('hardlink_master', True):
masters_preloaded.add(item.path)
@ -296,13 +296,13 @@ class DownloadPipeline:
# we only need to preload *once* (for the 1st selected slave)
chunks, _ = hardlink_masters[source]
if chunks is not None:
preload(chunks)
_preload(chunks)
masters_preloaded.add(source)
else:
# easy: we do not have a filter, thus all items are selected, thus we need to preload all chunks.
for item in items:
if 'chunks' in item:
preload(item.chunks)
_preload(item.chunks)
for item in items:
yield item