mount: Improve fuse filesystem memory efficiency

Closes #59

CHANGES

@@ -7,7 +7,8 @@ Version 0.12
 ------------
 
 (feature release, released on X)
-
+- ``attic mount`` now supports mounting an entire repository not only
+  individual archives (#59)
 - Added option to restrict remote repository access to specific path(s):
   ``attic serve --restrict-to-path X`` (#51)
 - Include "all archives" size information in "--stats" output. (#54)

attic/archiver.py

@@ -233,7 +233,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""")
         return self.exit_code
 
     def do_mount(self, args):
-        """Mount archive as a FUSE fileystem
+        """Mount archive or an entire repository as a FUSE fileystem
         """
         try:
             from attic.fuse import AtticOperations

attic/fuse.py

@@ -1,9 +1,11 @@
 from collections import defaultdict
 import errno
+import io
 import llfuse
 import msgpack
 import os
 import stat
+import tempfile
 import time
 from attic.archive import Archive
 from attic.helpers import daemonize
@@ -13,6 +15,21 @@ from attic.remote import cache_if_remote
 have_fuse_mtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
 
 
+class ItemCache:
+    def __init__(self):
+        self.fd = tempfile.TemporaryFile()
+        self.offset = 1000000
+
+    def add(self, item):
+        pos = self.fd.seek(0, io.SEEK_END)
+        self.fd.write(msgpack.packb(item))
+        return pos + self.offset
+
+    def get(self, inode):
+        self.fd.seek(inode - self.offset, io.SEEK_SET)
+        return next(msgpack.Unpacker(self.fd))
+
+
 class AtticOperations(llfuse.Operations):
     """Export Attic archive as a fuse filesystem
     """
@@ -26,12 +43,15 @@ class AtticOperations(llfuse.Operations):
         self.contents = defaultdict(dict)
         self.default_dir = {b'mode': 0o40755, b'mtime': int(time.time() * 1e9), b'uid': os.getuid(), b'gid': os.getgid()}
         self.pending_archives = {}
+        self.cache = ItemCache()
         if archive:
             self.process_archive(archive)
         else:
+            # Create root inode
             self.parent[1] = self.allocate_inode()
             self.items[1] = self.default_dir
             for archive_name in manifest.archives:
+                # Create archive placeholder inode
                 archive_inode = self.allocate_inode()
                 self.items[archive_inode] = self.default_dir
                 self.parent[archive_inode] = 1
@@ -60,10 +80,11 @@ class AtticOperations(llfuse.Operations):
                     if i == num_segments:
                         if b'source' in item and stat.S_ISREG(item[b'mode']):
                             inode = self._find_inode(item[b'source'], prefix)
-                            self.items[inode][b'nlink'] = self.items[inode].get(b'nlink', 1) + 1
-                        else:
-                            inode = self.allocate_inode()
+                            item = self.cache.get(inode)
+                            item[b'nlink'] = item.get(b'nlink', 1) + 1
                             self.items[inode] = item
+                        else:
+                            inode = self.cache.add(item)
                         self.parent[inode] = parent
                         if segment:
                             self.contents[parent][segment] = inode
@@ -93,6 +114,12 @@ class AtticOperations(llfuse.Operations):
         stat_.f_favail = 0
         return stat_
 
+    def get_item(self, inode):
+        try:
+            return self.items[inode]
+        except KeyError:
+            return self.cache.get(inode)
+
     def _find_inode(self, path, prefix=[]):
         segments = prefix + os.fsencode(os.path.normpath(path)).split(b'/')
         inode = 1
@@ -101,7 +128,7 @@ class AtticOperations(llfuse.Operations):
         return inode
 
     def getattr(self, inode):
-        item = self.items[inode]
+        item = self.get_item(inode)
         size = 0
         try:
             size = sum(size for _, size, _ in item[b'chunks'])
@@ -131,11 +158,11 @@ class AtticOperations(llfuse.Operations):
         return entry
 
     def listxattr(self, inode):
-        item = self.items[inode]
+        item = self.get_item(inode)
         return item.get(b'xattrs', {}).keys()
 
     def getxattr(self, inode, name):
-        item = self.items[inode]
+        item = self.get_item(inode)
         try:
             return item.get(b'xattrs', {})[name]
         except KeyError:
@@ -168,7 +195,7 @@ class AtticOperations(llfuse.Operations):
 
     def read(self, fh, offset, size):
         parts = []
-        item = self.items[fh]
+        item = self.get_item(fh)
         for id, s, csize in item[b'chunks']:
             if s < offset:
                 offset -= s
@@ -189,7 +216,8 @@ class AtticOperations(llfuse.Operations):
             yield name, self.getattr(inode), i + 1
 
     def readlink(self, inode):
-        return os.fsencode(self.items[inode][b'source'])
+        item = self.get_item(inode)
+        return os.fsencode(item[b'source'])
 
     def mount(self, mountpoint, extra_options, foreground=False):
         options = ['fsname=atticfs', 'ro']