From eabe7c7cc5be841e7e0c01f0d4b9e072347628bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Tue, 23 Jul 2013 13:40:54 +0200 Subject: [PATCH] More memory efficient implementation --- attic/fuse.py | 103 +++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 60 deletions(-) diff --git a/attic/fuse.py b/attic/fuse.py index 8ab6f3b3c..28373c6ad 100644 --- a/attic/fuse.py +++ b/attic/fuse.py @@ -11,79 +11,65 @@ class AtticOperations(llfuse.Operations): """ def __init__(self, key, repository, archive): super(AtticOperations, self).__init__() - print('__init__') + self._inode_count = 0 self.key = key self.repository = repository self.items = {} - self.inodes = {} - self.inode_parent = {} - self.inode_contents = defaultdict(dict) + self.parent = {} + self.contents = defaultdict(dict) + default_dir = {b'mode': 0o40755, b'mtime': 0, b'uid': os.getuid(), b'gid': os.getgid()} for item, _ in archive.iter_items(): head, tail = os.path.split(os.fsencode(os.path.normpath(item[b'path']))) segments = head.split(b'/') parent = 1 for segment in segments: - if not segment: - continue - if not segment in self.inode_contents[parent]: - node = self._make_directory_inode() - self.inodes[node.st_ino] = node - self.inode_parent[node.st_ino] = parent - self.inode_contents[parent][segment] = node.st_ino - parent = node.st_ino + if not segment in self.contents[parent]: + inode = self.allocate_inode() + self.items[inode] = default_dir + self.parent[inode] = parent + if segment: + self.contents[parent][segment] = inode + parent = inode else: - parent = self.inode_contents[parent][segment] + parent = self.contents[parent][segment] if b'source' in item and stat.S_ISREG(item[b'mode']): - node = self._find_inode(item[b'source']) + inode = self._find_inode(item[b'source']) + self.items[inode][b'nlink'] = self.items[inode].get(b'nlink', 1) + 1 else: - node = self._make_item_inode(item) - self.inodes[node.st_ino] = node - self.items[node.st_ino] = item - node.st_nlink += 1 - self.inode_parent[node.st_ino] = parent + inode = self.allocate_inode() + self.items[inode] = item + self.parent[inode] = parent if tail: - self.inode_contents[parent][tail] = node.st_ino + self.contents[parent][tail] = inode + else: + self.items[inode] = default_dir + + def allocate_inode(self): + self._inode_count += 1 + return self._inode_count def _find_inode(self, path): segments = os.fsencode(os.path.normpath(path)).split(b'/') inode = 1 for segment in segments: - inode = self.inode_contents[inode][segment] - return self.inodes[inode] + inode = self.contents[inode][segment] + return inode - def _make_directory_inode(self): - entry = llfuse.EntryAttributes() - entry.st_ino = len(self.inodes) + 1 - entry.generation = 0 - entry.entry_timeout = 300 - entry.attr_timeout = 300 - entry.st_mode = 0o40755 - entry.st_nlink = 0 - entry.st_uid = os.getuid() - entry.st_gid = os.getgid() - entry.st_rdev = 0 - entry.st_size = 0 - entry.st_blksize = 512 - entry.st_blocks = 1 - entry.st_atime = time.time() - entry.st_mtime = time.time() - entry.st_ctime = time.time() - return entry - - def _make_item_inode(self, item): + def getattr(self, inode): + item = self.items[inode] size = 0 try: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass entry = llfuse.EntryAttributes() - entry.st_ino = len(self.inodes) + 1 + entry.st_ino = inode entry.generation = 0 entry.entry_timeout = 300 entry.attr_timeout = 300 entry.st_mode = item[b'mode'] - entry.st_nlink = 0 + entry.st_nlink = item.get(b'nlink', 1) entry.st_uid = item[b'uid'] entry.st_gid = item[b'uid'] entry.st_rdev = item.get(b'rdev', 0) @@ -95,18 +81,6 @@ def _make_item_inode(self, item): entry.st_ctime = item[b'mtime'] / 1e9 return entry - def run(self, dir): - llfuse.init(self, dir, ['fsname=atticfs', 'nonempty']) - try: - llfuse.main(single=True) - except: - llfuse.close(unmount=False) - raise - llfuse.close() - - def getattr(self, inode): - return self.inodes[inode] - def listxattr(self, inode): item = self.items[inode] return [b'user.' + name for name in item.get(b'xattrs', {}).keys()] @@ -124,9 +98,9 @@ def lookup(self, parent_inode, name): if name == b'.': inode = parent_inode elif name == b'..': - inode = self.inode_parent[parent_inode] + inode = self.parent[parent_inode] else: - inode = self.inode_contents[parent_inode].get(name) + inode = self.contents[parent_inode].get(name) if not inode: raise llfuse.FUSEError(errno.ENOENT) @@ -155,10 +129,19 @@ def read(self, fh, offset, size): return b''.join(parts) def readdir(self, fh, off): - entries = [(b'.', fh), (b'..', self.inode_parent[fh])] - entries.extend(self.inode_contents[fh].items()) + entries = [(b'.', fh), (b'..', self.parent[fh])] + entries.extend(self.contents[fh].items()) for i, (name, inode) in enumerate(entries[off:], off): yield name, self.getattr(inode), i + 1 def readlink(self, inode): return os.fsencode(self.items[inode][b'source']) + + def run(self, dir): + llfuse.init(self, dir, ['fsname=atticfs', 'nonempty']) + try: + llfuse.main(single=True) + except: + llfuse.close() + raise + llfuse.close()