mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

fuse: update comments

This commit is contained in:
Marian Beermann 2017-06-15 23:50:17 +02:00
parent 3b928a4558
commit 2766693706
1 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
src/borg/fuse.py
View File

@ -43,8 +43,9 @@ class ItemCache:
and retrieves items from these inode numbers.
"""
# Approximately ~230000 items (depends on the average number of items per metadata chunk)
# Since growing a bytearray has to copy it, growing it will converge to o(n), however,
# 2 MiB are approximately ~230000 items (depends on the average number of items per metadata chunk).
#
# Since growing a bytearray has to copy it, growing it will converge to O(n^2), however,
# this is not yet relevant due to the swiftness of copying memory. If it becomes an issue,
# use an anonymous mmap and just resize that (or, if on 64 bit, make it so big you never need
# to resize it in the first place; that's free).
@ -58,32 +59,35 @@ class ItemCache:
# self.meta, the "meta-array" is a densely packed array of metadata about where items can be found.
# It is indexed by the inode number minus self.offset. (This is in a way eerily similar to how the first
# unices did this).
# The meta-array contains chunk IDs and item entries (described in inode_for_current_item).
# The meta-array contains chunk IDs and item entries (described in iter_archive_items).
# The chunk IDs are referenced by item entries through relative offsets,
# which are bounded by the metadata chunk size.
self.meta = bytearray()
# The current write offset in self.meta
self.write_offset = 0
# Offset added to meta-indices, resulting in an inode,
# or substracted from inodes, resulting in a meta-indices.
# Offset added to meta-indices, resulting in inodes,
# or subtracted from inodes, resulting in meta-indices.
# XXX: Merge FuseOperations.items and ItemCache to avoid
# this implicit limitation / hack (on the number of synthetic inodes, degenerate
# cases can inflate their number far beyond the number of archives).
self.offset = 1000000
# A temporary file that contains direct items, i.e. items directly cached in this layer.
# These are items that span more than one chunk and thus cannot be efficiently cached
# by the object cache (self.decrypted_repository), which would require variable-length structures;
# possible but not worth the effort, see inode_for_current_item.
# possible but not worth the effort, see iter_archive_items.
self.fd = tempfile.TemporaryFile(prefix='borg-tmp')
# A small LRU cache for chunks requested by ItemCache.get() from the object cache,
# this significantly speeds up directory traversal and similar operations which
# tend to re-read the same chunks over and over.
# The capacity is kept low because increasing it does not provide any significant advantage,
# but makes LRUCache's square behaviour noticeable as well as consuming some memory.
# but makes LRUCache's square behaviour noticeable and consumes more memory.
self.chunks = LRUCache(capacity=10, dispose=lambda _: None)
# Instrumentation
# Count of indirect items, i.e. data is cached in the object cache, in this cache
# Count of indirect items, i.e. data is cached in the object cache, not directly in this cache
self.indirect_items = 0
# Count of direct items, i.e. data is in self.fd
self.direct_items = 0
@ -92,16 +96,11 @@ class ItemCache:
offset = inode - self.offset
if offset < 0:
raise ValueError('ItemCache.get() called with an invalid inode number')
if self.meta[offset] == ord(b'S'):
fd_offset = int.from_bytes(self.meta[offset + 1:offset + 9], 'little')
self.fd.seek(fd_offset, io.SEEK_SET)
return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024)))
else:
if self.meta[offset] == ord(b'I'):
_, chunk_id_relative_offset, chunk_offset = self.indirect_entry_struct.unpack_from(self.meta, offset)
chunk_id_offset = offset - chunk_id_relative_offset
# bytearray slices are bytearrays as well, explicitly convert to bytes()
chunk_id = bytes(self.meta[chunk_id_offset:chunk_id_offset + 32])
chunk_offset = int.from_bytes(self.meta[offset + 5:offset + 9], 'little')
chunk = self.chunks.get(chunk_id)
if not chunk:
csize, chunk = next(self.decrypted_repository.get_many([chunk_id]))
@ -110,12 +109,21 @@ class ItemCache:
unpacker = msgpack.Unpacker()
unpacker.feed(data)
return Item(internal_dict=next(unpacker))
elif self.meta[offset] == ord(b'S'):
fd_offset = int.from_bytes(self.meta[offset + 1:offset + 9], 'little')
self.fd.seek(fd_offset, io.SEEK_SET)
return Item(internal_dict=next(msgpack.Unpacker(self.fd, read_size=1024)))
else:
raise ValueError('Invalid entry type in self.meta')
def iter_archive_items(self, archive_item_ids):
unpacker = msgpack.Unpacker()
# Current offset in the metadata stream, which consists of all metadata chunks glued together
stream_offset = 0
# Offset of the current chunk in the metadata stream
chunk_begin = 0
# Length of the chunk preciding the current chunk
last_chunk_length = 0
msgpacked_bytes = b''
@ -124,6 +132,7 @@ class ItemCache:
pack_indirect_into = self.indirect_entry_struct.pack_into
def write_bytes(append_msgpacked_bytes):
# XXX: Future versions of msgpack include an Unpacker.tell() method that provides this for free.
nonlocal msgpacked_bytes
nonlocal stream_offset
msgpacked_bytes += append_msgpacked_bytes
@ -150,9 +159,9 @@ class ItemCache:
# Need more data, feed the next chunk
break
current_item_length = len(msgpacked_bytes)
current_spans_chunks = stream_offset - current_item_length <= chunk_begin
current_item = msgpacked_bytes
current_item_length = len(current_item)
current_spans_chunks = stream_offset - current_item_length < chunk_begin
msgpacked_bytes = b''
if write_offset + 9 >= len(meta):
@ -178,15 +187,13 @@ class ItemCache:
pos = self.fd.seek(0, io.SEEK_END)
self.fd.write(current_item)
meta[write_offset:write_offset + 9] = b'S' + pos.to_bytes(8, 'little')
write_offset += 9
self.direct_items += 1
inode = write_offset - 9 + self.offset
else:
item_offset = stream_offset - current_item_length - chunk_begin
pack_indirect_into(meta, write_offset, b'I', write_offset - current_id_offset, item_offset)
write_offset += 9
self.indirect_items += 1
inode = write_offset - 9 + self.offset
inode = write_offset + self.offset
write_offset += 9
yield inode, Item(internal_dict=item)