diff --git a/src/borg/cache.py b/src/borg/cache.py index 21efcbc9e..cc57e7bc9 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -20,13 +20,12 @@ from .helpers import format_file_size from .helpers import yes from .helpers import remove_surrogates from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage -from .item import Item, ArchiveItem +from .item import Item, ArchiveItem, ChunkListEntry from .key import PlaintextKey from .locking import Lock from .platform import SaveFile from .remote import cache_if_remote -ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f6247cd38..2bd5f4071 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1701,10 +1701,12 @@ class ItemFormatter(BaseFormatter): return len(item.get('chunks', [])) def calculate_size(self, item): - return item.file_size() + # note: does not support hardlink slaves, they will be size 0 + return item.file_size(compressed=False) def calculate_csize(self, item): - return sum(c.csize for c in item.get('chunks', [])) + # note: does not support hardlink slaves, they will be csize 0 + return item.file_size(compressed=True) def hash_item(self, hash_function, item): if 'chunks' not in item: diff --git a/src/borg/item.pyx b/src/borg/item.pyx index a0b9e3efd..c3125c577 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -1,3 +1,5 @@ +from collections import namedtuple + from .constants import ITEM_KEYS from .helpers import safe_encode, safe_decode from .helpers import StableDict @@ -113,6 +115,8 @@ class PropDict: return property(_get, _set, _del, doc=doc) +ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') + class Item(PropDict): """ Item abstraction that deals with validation and the low-level details internally: @@ -172,23 +176,38 @@ class Item(PropDict): part = PropDict._make_property('part', int) - def file_size(self, hardlink_masters=None, memorize=False): - """determine the size of this item""" - size = self.get('size') - if size is not None: - return size - chunks = self.get('chunks') - having_chunks = chunks is not None - if not having_chunks: - # this item has no (own) chunks, but if this is a hardlink slave - # and we know the master, we can still compute the size. - hardlink_masters = hardlink_masters or {} - chunks, _ = hardlink_masters.get(self.get('source'), (None, None)) - if chunks is None: - return 0 - size = sum(chunk.size for chunk in chunks) - if memorize and having_chunks: - self.size = size + def file_size(self, hardlink_masters=None, memorize=False, compressed=False): + """determine the (uncompressed or compressed) size of this item""" + attr = 'csize' if compressed else 'size' + try: + size = getattr(self, attr) + except AttributeError: + # no precomputed (c)size value available, compute it: + try: + chunks = getattr(self, 'chunks') + having_chunks = True + except AttributeError: + having_chunks = False + # this item has no (own) chunks list, but if this is a hardlink slave + # and we know the master, we can still compute the size. + if hardlink_masters is None: + chunks = None + else: + try: + master = getattr(self, 'source') + except AttributeError: + # not a hardlink slave, likely a directory or special file w/o chunks + chunks = None + else: + # hardlink slave, try to fetch hardlink master's chunks list + # todo: put precomputed size into hardlink_masters' values and use it, if present + chunks, _ = hardlink_masters.get(master, (None, None)) + if chunks is None: + return 0 + size = sum(getattr(ChunkListEntry(*chunk), attr) for chunk in chunks) + # if requested, memorize the precomputed (c)size for items that have an own chunks list: + if memorize and having_chunks: + setattr(self, attr, size) return size