From 2493598eefd63a9f2ea63515f2d180a88460e6d1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 20 Jan 2018 06:36:54 +0100 Subject: [PATCH 1/2] files cache: improve exception handling, fixes #3553 now deals with: - corrupted files cache (truncated or modified not by borg) - inaccessible/unreadable files cache - missing files cache The latter fix is not sufficient, the cache transaction processing would still stumble over expected, but missing files in the cache. (cherry picked from commit 423ec4ba1e5d4e3c79358e1cd90ff5aca0da06d1) --- src/borg/cache.py | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index 82329153f..d2d8a68cb 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -502,25 +502,32 @@ class LocalCache(CacheStatsMixin): self.files = {} self._newest_cmtime = None logger.debug('Reading files cache ...') - - with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False, - integrity_data=self.cache_config.integrity.get('files')) as fd: - u = msgpack.Unpacker(use_list=True) - while True: - data = fd.read(64 * 1024) - if not data: - break - u.feed(data) - try: - for path_hash, item in u: - entry = FileCacheEntry(*item) - # in the end, this takes about 240 Bytes per file - self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1)) - except (TypeError, ValueError) as exc: - logger.warning('The files cache seems corrupt, ignoring it. ' - 'Expect lower performance. [%s]' % str(exc)) - self.files = {} - return + msg = None + try: + with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False, + integrity_data=self.cache_config.integrity.get('files')) as fd: + u = msgpack.Unpacker(use_list=True) + while True: + data = fd.read(64 * 1024) + if not data: + break + u.feed(data) + try: + for path_hash, item in u: + entry = FileCacheEntry(*item) + # in the end, this takes about 240 Bytes per file + self.files[path_hash] = msgpack.packb(entry._replace(age=entry.age + 1)) + except (TypeError, ValueError) as exc: + msg = "The files cache seems invalid. [%s]" % str(exc) + break + except OSError as exc: + msg = "The files cache can't be read. [%s]" % str(exc) + except FileIntegrityError as fie: + msg = "The files cache is corrupted. [%s]" % str(fie) + if msg is not None: + logger.warning(msg) + logger.warning('Continuing without files cache - expect lower performance.') + self.files = {} def begin_txn(self): # Initialize transaction snapshot From 835fede85f15956a5b9650f33790c7e665950ca1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 20 Jan 2018 13:34:46 +0100 Subject: [PATCH 2/2] fixup: tests --- src/borg/testsuite/archiver.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 7b6e95b11..086002c96 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3325,13 +3325,9 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase): def test_cache_files(self): self.cmd('create', self.repository_location + '::test', 'input') self.corrupt(os.path.join(self.cache_path, 'files')) - - if self.FORK_DEFAULT: - out = self.cmd('create', self.repository_location + '::test1', 'input', exit_code=2) - assert 'failed integrity check' in out - else: - with pytest.raises(FileIntegrityError): - self.cmd('create', self.repository_location + '::test1', 'input') + out = self.cmd('create', self.repository_location + '::test1', 'input') + # borg warns about the corrupt files cache, but then continues without files cache. + assert 'files cache is corrupted' in out def test_chunks_archive(self): self.cmd('create', self.repository_location + '::test1', 'input')