From 2815ba51fa93135c9c8574c8789f5d4bb2c2e05c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2020 16:11:30 +0100 Subject: [PATCH 1/3] use same code in export-tar as in extract the partial_extract parameter was missing and that can lead to to many items being preloaded. --- src/borg/archiver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 254e1bae..c2e49dfa 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1115,7 +1115,8 @@ class Archiver: return None, stream return tarinfo, stream - for item in archive.iter_items(filter, preload=True, hardlink_masters=hardlink_masters): + for item in archive.iter_items(filter, partial_extract=partial_extract, + preload=True, hardlink_masters=hardlink_masters): orig_path = item.path if strip_components: item.path = os.sep.join(orig_path.split(os.sep)[strip_components:]) From 6a18c47464e06518d53ae6080b1e842059af184f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 15 Dec 2020 18:08:54 +0100 Subject: [PATCH 2/3] cosmetic: make parens structure in borg export-tar similar to borg extract --- src/borg/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index c2e49dfa..9d2731e3 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1013,8 +1013,8 @@ class Archiver: hardlink_masters = {} if partial_extract else None def peek_and_store_hardlink_masters(item, matched): - if (partial_extract and not matched and hardlinkable(item.mode) and - item.get('hardlink_master', True) and 'source' not in item): + if ((partial_extract and not matched and hardlinkable(item.mode)) and + (item.get('hardlink_master', True) and 'source' not in item)): hardlink_masters[item.get('path')] = (item.get('chunks'), None) filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) From 24d3400dd455bf78a9f54d219111e7b2d5468521 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 17 Dec 2020 22:28:42 +0100 Subject: [PATCH 3/3] borg export-tar: fix memory leak with ssh: remote repository, fixes #5568 also: added a comment how to avoid this kind of memory leak. --- src/borg/archive.py | 2 ++ src/borg/archiver.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 7c0369d6..708d78b1 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -523,6 +523,8 @@ Utilization of max. archive size: {csize_max:.0%} return filter(item) if filter else True def iter_items(self, filter=None, partial_extract=False, preload=False, hardlink_masters=None): + # note: when calling this with preload=True, later fetch_many() must be called with + # is_preloaded=True or the RemoteRepository code will leak memory! assert not (filter and partial_extract and preload) or hardlink_masters is not None for item in self.pipeline.unpack_many(self.metadata.items, partial_extract=partial_extract, preload=preload, hardlink_masters=hardlink_masters, diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 9d2731e3..3c7d45e1 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1031,7 +1031,8 @@ class Archiver: """ Return a file-like object that reads from the chunks of *item*. """ - chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _, _ in item.chunks]) + chunk_iterator = archive.pipeline.fetch_many([chunk_id for chunk_id, _, _ in item.chunks], + is_preloaded=True) if pi: info = [remove_surrogates(item.path)] return ChunkIteratorFileWrapper(chunk_iterator,