mirror of https://github.com/borgbackup/borg.git
Merge pull request #4542 from ThomasWaldmann/fix-extract-hardlinks-1.1
slave hardlinks extraction issue, see #4350 (1.1-maint)
This commit is contained in:
commit
b033693c9a
|
@ -192,7 +192,7 @@ class DownloadPipeline:
|
||||||
self.repository = repository
|
self.repository = repository
|
||||||
self.key = key
|
self.key = key
|
||||||
|
|
||||||
def unpack_many(self, ids, filter=None, preload=False):
|
def unpack_many(self, ids, filter=None, partial_extract=False, preload=False, hardlink_masters=None):
|
||||||
"""
|
"""
|
||||||
Return iterator of items.
|
Return iterator of items.
|
||||||
|
|
||||||
|
@ -209,12 +209,40 @@ class DownloadPipeline:
|
||||||
for item in items:
|
for item in items:
|
||||||
if 'chunks' in item:
|
if 'chunks' in item:
|
||||||
item.chunks = [ChunkListEntry(*e) for e in item.chunks]
|
item.chunks = [ChunkListEntry(*e) for e in item.chunks]
|
||||||
|
|
||||||
|
def preload(chunks):
|
||||||
|
self.repository.preload([c.id for c in chunks])
|
||||||
|
|
||||||
if filter:
|
if filter:
|
||||||
items = [item for item in items if filter(item)]
|
items = [item for item in items if filter(item)]
|
||||||
|
|
||||||
if preload:
|
if preload:
|
||||||
for item in items:
|
if filter and partial_extract:
|
||||||
if 'chunks' in item:
|
# if we do only a partial extraction, it gets a bit
|
||||||
self.repository.preload([c.id for c in item.chunks])
|
# complicated with computing the preload items: if a hardlink master item is not
|
||||||
|
# selected (== not extracted), we will still need to preload its chunks if a
|
||||||
|
# corresponding hardlink slave is selected (== is extracted).
|
||||||
|
# due to a side effect of the filter() call, we now have hardlink_masters dict populated.
|
||||||
|
masters_preloaded = set()
|
||||||
|
for item in items:
|
||||||
|
if 'chunks' in item: # regular file, maybe a hardlink master
|
||||||
|
preload(item.chunks)
|
||||||
|
# if this is a hardlink master, remember that we already preloaded it:
|
||||||
|
if 'source' not in item and hardlinkable(item.mode) and item.get('hardlink_master', True):
|
||||||
|
masters_preloaded.add(item.path)
|
||||||
|
elif 'source' in item and hardlinkable(item.mode): # hardlink slave
|
||||||
|
source = item.source
|
||||||
|
if source not in masters_preloaded:
|
||||||
|
# we only need to preload *once* (for the 1st selected slave)
|
||||||
|
chunks, _ = hardlink_masters[source]
|
||||||
|
preload(chunks)
|
||||||
|
masters_preloaded.add(source)
|
||||||
|
else:
|
||||||
|
# easy: we do not have a filter, thus all items are selected, thus we need to preload all chunks.
|
||||||
|
for item in items:
|
||||||
|
if 'chunks' in item:
|
||||||
|
preload(item.chunks)
|
||||||
|
|
||||||
for item in items:
|
for item in items:
|
||||||
yield item
|
yield item
|
||||||
|
|
||||||
|
@ -433,8 +461,10 @@ Utilization of max. archive size: {csize_max:.0%}
|
||||||
return False
|
return False
|
||||||
return filter(item) if filter else True
|
return filter(item) if filter else True
|
||||||
|
|
||||||
def iter_items(self, filter=None, preload=False):
|
def iter_items(self, filter=None, partial_extract=False, preload=False, hardlink_masters=None):
|
||||||
for item in self.pipeline.unpack_many(self.metadata.items, preload=preload,
|
assert not (filter and partial_extract and preload) or hardlink_masters is not None
|
||||||
|
for item in self.pipeline.unpack_many(self.metadata.items, partial_extract=partial_extract,
|
||||||
|
preload=preload, hardlink_masters=hardlink_masters,
|
||||||
filter=lambda item: self.item_filter(item, filter)):
|
filter=lambda item: self.item_filter(item, filter)):
|
||||||
yield item
|
yield item
|
||||||
|
|
||||||
|
|
|
@ -755,7 +755,8 @@ class Archiver:
|
||||||
else:
|
else:
|
||||||
pi = None
|
pi = None
|
||||||
|
|
||||||
for item in archive.iter_items(filter, preload=True):
|
for item in archive.iter_items(filter, partial_extract=partial_extract,
|
||||||
|
preload=True, hardlink_masters=hardlink_masters):
|
||||||
orig_path = item.path
|
orig_path = item.path
|
||||||
if strip_components:
|
if strip_components:
|
||||||
item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
|
item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
|
||||||
|
@ -997,7 +998,7 @@ class Archiver:
|
||||||
return None, stream
|
return None, stream
|
||||||
return tarinfo, stream
|
return tarinfo, stream
|
||||||
|
|
||||||
for item in archive.iter_items(filter, preload=True):
|
for item in archive.iter_items(filter, preload=True, hardlink_masters=hardlink_masters):
|
||||||
orig_path = item.path
|
orig_path = item.path
|
||||||
if strip_components:
|
if strip_components:
|
||||||
item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
|
item.path = os.sep.join(orig_path.split(os.sep)[strip_components:])
|
||||||
|
|
|
@ -823,7 +823,18 @@ class ArchiverTestCase(ArchiverTestCaseBase):
|
||||||
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
||||||
|
|
||||||
@requires_hardlinks
|
@requires_hardlinks
|
||||||
def test_extract_hardlinks(self):
|
def test_extract_hardlinks1(self):
|
||||||
|
self._extract_hardlinks_setup()
|
||||||
|
with changedir('output'):
|
||||||
|
self.cmd('extract', self.repository_location + '::test')
|
||||||
|
assert os.stat('input/source').st_nlink == 4
|
||||||
|
assert os.stat('input/abba').st_nlink == 4
|
||||||
|
assert os.stat('input/dir1/hardlink').st_nlink == 4
|
||||||
|
assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4
|
||||||
|
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
||||||
|
|
||||||
|
@requires_hardlinks
|
||||||
|
def test_extract_hardlinks2(self):
|
||||||
self._extract_hardlinks_setup()
|
self._extract_hardlinks_setup()
|
||||||
with changedir('output'):
|
with changedir('output'):
|
||||||
self.cmd('extract', self.repository_location + '::test', '--strip-components', '2')
|
self.cmd('extract', self.repository_location + '::test', '--strip-components', '2')
|
||||||
|
@ -839,13 +850,6 @@ class ArchiverTestCase(ArchiverTestCaseBase):
|
||||||
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
||||||
assert os.stat('input/dir1/aaaa').st_nlink == 2
|
assert os.stat('input/dir1/aaaa').st_nlink == 2
|
||||||
assert os.stat('input/dir1/source2').st_nlink == 2
|
assert os.stat('input/dir1/source2').st_nlink == 2
|
||||||
with changedir('output'):
|
|
||||||
self.cmd('extract', self.repository_location + '::test')
|
|
||||||
assert os.stat('input/source').st_nlink == 4
|
|
||||||
assert os.stat('input/abba').st_nlink == 4
|
|
||||||
assert os.stat('input/dir1/hardlink').st_nlink == 4
|
|
||||||
assert os.stat('input/dir1/subdir/hardlink').st_nlink == 4
|
|
||||||
assert open('input/dir1/subdir/hardlink', 'rb').read() == b'123456'
|
|
||||||
|
|
||||||
def test_extract_include_exclude(self):
|
def test_extract_include_exclude(self):
|
||||||
self.cmd('init', '--encryption=repokey', self.repository_location)
|
self.cmd('init', '--encryption=repokey', self.repository_location)
|
||||||
|
|
Loading…
Reference in New Issue