Merge branch 'lrucache' of https://github.com/sourcejedi/borg

borg/lrucache.py

@@ -1,42 +1,41 @@
-class LRUCache(dict):
+class LRUCache:
-
+    def __init__(self, capacity, dispose):
-    def __init__(self, capacity):
+        self._cache = {}
-        super().__init__()
         self._lru = []
         self._capacity = capacity
+        self._dispose = dispose
 
     def __setitem__(self, key, value):
-        try:
+        assert key not in self._cache, (
-            self._lru.remove(key)
+            "Unexpected attempt to replace a cached item,"
-        except ValueError:
+            " without first deleting the old item.")
-            pass
         self._lru.append(key)
         while len(self._lru) > self._capacity:
             del self[self._lru[0]]
-        return super().__setitem__(key, value)
+        self._cache[key] = value
 
     def __getitem__(self, key):
-        try:
+        value = self._cache[key]  # raise KeyError if not found
-            self._lru.remove(key)
+        self._lru.remove(key)
-            self._lru.append(key)
+        self._lru.append(key)
-        except ValueError:
+        return value
-            pass
-        return super().__getitem__(key)
 
     def __delitem__(self, key):
-        try:
+        value = self._cache.pop(key)  # raise KeyError if not found
-            self._lru.remove(key)
+        self._dispose(value)
-        except ValueError:
+        self._lru.remove(key)
-            pass
-        return super().__delitem__(key)
 
-    def pop(self, key, default=None):
+    def __contains__(self, key):
-        try:
+        return key in self._cache
-            self._lru.remove(key)
-        except ValueError:
-            pass
-        return super().pop(key, default)
 
-    def _not_implemented(self, *args, **kw):
+    def clear(self):
-        raise NotImplementedError
+        for value in self._cache.values():
-    popitem = setdefault = update = _not_implemented
+            self._dispose(value)
+        self._cache.clear()
+
+    # useful for testing
+    def items(self):
+        return self._cache.items()
+
+    def __len__(self):
+        return len(self._cache)

borg/repository.py

@@ -432,7 +432,8 @@ class LoggedIO:
 
     def __init__(self, path, limit, segments_per_dir, capacity=90):
         self.path = path
-        self.fds = LRUCache(capacity)
+        self.fds = LRUCache(capacity,
+                            dispose=lambda fd: fd.close())
         self.segment = 0
         self.limit = limit
         self.segments_per_dir = segments_per_dir
@@ -440,9 +441,8 @@ def __init__(self, path, limit, segments_per_dir, capacity=90):
         self._write_fd = None
 
     def close(self):
-        for segment in list(self.fds.keys()):
-            self.fds.pop(segment).close()
         self.close_segment()
+        self.fds.clear()
         self.fds = None  # Just to make sure we're disabled
 
     def segment_iterator(self, reverse=False):
@@ -516,9 +516,8 @@ def get_fd(self, segment):
             return fd
 
     def delete_segment(self, segment):
-        fd = self.fds.pop(segment)
+        if segment in self.fds:
-        if fd is not None:
+            del self.fds[segment]
-            fd.close()
         try:
             os.unlink(self.segment_filename(segment))
         except OSError:
@@ -561,9 +560,8 @@ def iter_objects(self, segment, include_data=False):
             header = fd.read(self.header_fmt.size)
 
     def recover_segment(self, segment, filename):
-        fd = self.fds.pop(segment)
+        if segment in self.fds:
-        if fd is not None:
+            del self.fds[segment]
-            fd.close()
         # FIXME: save a copy of the original file
         with open(filename, 'rb') as fd:
             data = memoryview(fd.read())

borg/testsuite/lrucache.py

@@ -1,40 +1,52 @@
 from ..lrucache import LRUCache
-from . import BaseTestCase
+import pytest
+from tempfile import TemporaryFile
 
 
-class LRUCacheTestCase(BaseTestCase):
+class TestLRUCache:
 
-    def test(self):
+    def test_lrucache(self):
-        c = LRUCache(2)
+        c = LRUCache(2, dispose=lambda _: None)
-        self.assert_equal(len(c), 0)
+        assert len(c) == 0
+        assert c.items() == set()
         for i, x in enumerate('abc'):
             c[x] = i
-        self.assert_equal(len(c), 2)
+        assert len(c) == 2
-        self.assert_equal(set(c), set(['b', 'c']))
+        assert c.items() == set([('b', 1), ('c', 2)])
-        self.assert_equal(set(c.items()), set([('b', 1), ('c', 2)]))
+        assert 'a' not in c
-        self.assert_equal(False, 'a' in c)
+        assert 'b' in c
-        self.assert_equal(True, 'b' in c)
+        with pytest.raises(KeyError):
-        self.assert_raises(KeyError, lambda: c['a'])
+            c['a']
-        self.assert_equal(c['b'], 1)
+        assert c['b'] == 1
-        self.assert_equal(c['c'], 2)
+        assert c['c'] == 2
         c['d'] = 3
-        self.assert_equal(len(c), 2)
+        assert len(c) == 2
-        self.assert_equal(c['c'], 2)
+        assert c['c'] == 2
-        self.assert_equal(c['d'], 3)
+        assert c['d'] == 3
-        c['c'] = 22
-        c['e'] = 4
-        self.assert_equal(len(c), 2)
-        self.assert_raises(KeyError, lambda: c['d'])
-        self.assert_equal(c['c'], 22)
-        self.assert_equal(c['e'], 4)
         del c['c']
-        self.assert_equal(len(c), 1)
+        assert len(c) == 1
-        self.assert_raises(KeyError, lambda: c['c'])
+        with pytest.raises(KeyError):
-        self.assert_equal(c['e'], 4)
+            c['c']
+        assert c['d'] == 3
+        c.clear()
+        assert c.items() == set()
 
-    def test_pop(self):
+    def test_dispose(self):
-        c = LRUCache(2)
+        c = LRUCache(2, dispose=lambda f: f.close())
-        c[1] = 1
+        f1 = TemporaryFile()
-        c[2] = 2
+        f2 = TemporaryFile()
-        c.pop(1)
+        f3 = TemporaryFile()
-        c[3] = 3
+        c[1] = f1
+        c[2] = f2
+        assert not f2.closed
+        c[3] = f3
+        assert 1 not in c
+        assert f1.closed
+        assert 2 in c
+        assert not f2.closed
+        del c[2]
+        assert 2 not in c
+        assert f2.closed
+        c.clear()
+        assert not c.items()
+        assert f3.closed