cache: extract CacheConfig class

src/borg/cache.py

@@ -35,6 +35,7 @@ class SecurityManager:
     def __init__(self, repository):
         self.repository = repository
         self.dir = get_security_dir(repository.id_str)
+        self.cache_dir = cache_dir(repository)
         self.key_type_file = os.path.join(self.dir, 'key-type')
         self.location_file = os.path.join(self.dir, 'location')
         self.manifest_ts_file = os.path.join(self.dir, 'manifest-timestamp')
@@ -52,9 +53,9 @@ def key_matches(self, key):
         except OSError as exc:
             logger.warning('Could not read/parse key type file: %s', exc)
 
-    def save(self, manifest, key, cache):
+    def save(self, manifest, key):
         logger.debug('security: saving state for %s to %s', self.repository.id_str, self.dir)
-        current_location = cache.repository._location.canonical_path()
+        current_location = self.repository._location.canonical_path()
         logger.debug('security: current location   %s', current_location)
         logger.debug('security: key type           %s', str(key.TYPE))
         logger.debug('security: manifest timestamp %s', manifest.timestamp)
@@ -65,7 +66,7 @@ def save(self, manifest, key, cache):
         with open(self.manifest_ts_file, 'w') as fd:
             fd.write(manifest.timestamp)
 
-    def assert_location_matches(self, cache):
+    def assert_location_matches(self, cache_config):
         # Warn user before sending data to a relocated repository
         try:
             with open(self.location_file) as fd:
@@ -77,13 +78,15 @@ def assert_location_matches(self, cache):
         except OSError as exc:
             logger.warning('Could not read previous location file: %s', exc)
             previous_location = None
-        if cache.previous_location and previous_location != cache.previous_location:
+        if cache_config.previous_location and previous_location != cache_config.previous_location:
             # Reconcile cache and security dir; we take the cache location.
-            previous_location = cache.previous_location
+            previous_location = cache_config.previous_location
             logger.debug('security: using previous_location of cache: %r', previous_location)
-        if previous_location and previous_location != self.repository._location.canonical_path():
+
+        repository_location = self.repository._location.canonical_path()
+        if previous_location and previous_location != repository_location:
             msg = ("Warning: The repository at location {} was previously located at {}\n".format(
-                self.repository._location.canonical_path(), previous_location) +
+                repository_location, previous_location) +
                 "Do you want to continue? [yN] ")
             if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
                        retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'):
@@ -91,11 +94,10 @@ def assert_location_matches(self, cache):
             # adapt on-disk config immediately if the new location was accepted
             logger.debug('security: updating location stored in cache and security dir')
             with open(self.location_file, 'w') as fd:
-                fd.write(cache.repository._location.canonical_path())
+                fd.write(repository_location)
-            cache.begin_txn()
+            cache_config.save()
-            cache.commit()
 
-    def assert_no_manifest_replay(self, manifest, key, cache):
+    def assert_no_manifest_replay(self, manifest, key, cache_config):
         try:
             with open(self.manifest_ts_file) as fd:
                 timestamp = fd.read()
@@ -106,7 +108,7 @@ def assert_no_manifest_replay(self, manifest, key, cache):
         except OSError as exc:
             logger.warning('Could not read previous location file: %s', exc)
             timestamp = ''
-        timestamp = max(timestamp, cache.timestamp or '')
+        timestamp = max(timestamp, cache_config.timestamp or '')
         logger.debug('security: determined newest manifest timestamp as %s', timestamp)
         # If repository is older than the cache or security dir something fishy is going on
         if timestamp and timestamp > manifest.timestamp:
@@ -122,22 +124,136 @@ def assert_key_type(self, key, cache):
         if self.known() and not self.key_matches(key):
             raise Cache.EncryptionMethodMismatch()
 
-    def assert_secure(self, manifest, key, cache):
+    def assert_secure(self, manifest, key, *, cache_config=None, warn_if_unencrypted=True):
-        self.assert_location_matches(cache)
+        self.assert_access_unknown(warn_if_unencrypted, manifest, key)
-        self.assert_key_type(key, cache)
+        if cache_config:
-        self.assert_no_manifest_replay(manifest, key, cache)
+            self._assert_secure(manifest, key, cache_config)
-        if not self.known():
+        else:
-            self.save(manifest, key, cache)
+            with CacheConfig(self.repository):
+                self._assert_secure(manifest, key, cache_config)
 
-    def assert_access_unknown(self, warn_if_unencrypted, key):
+    def _assert_secure(self, manifest, key, cache_config):
-        if warn_if_unencrypted and not key.logically_encrypted and not self.known():
+        self.assert_location_matches(cache_config)
+        self.assert_key_type(key, cache_config)
+        self.assert_no_manifest_replay(manifest, key, cache_config)
+        if not self.known():
+            logger.debug('security: saving state for previously unknown repository')
+            self.save(manifest, key)
+
+    def assert_access_unknown(self, warn_if_unencrypted, manifest, key):
+        if not key.logically_encrypted and not self.known():
             msg = ("Warning: Attempting to access a previously unknown unencrypted repository!\n" +
                    "Do you want to continue? [yN] ")
-            if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.",
+            allow_access = not warn_if_unencrypted or yes(msg, false_msg="Aborting.",
-                       retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'):
+                invalid_msg="Invalid answer, aborting.",
+                retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK')
+            if allow_access:
+                if warn_if_unencrypted:
+                    logger.debug('security: saving state for unknown unencrypted repository (explicitly granted)')
+                else:
+                    logger.debug('security: saving state for unknown unencrypted repository')
+                self.save(manifest, key)
+            else:
                 raise Cache.CacheInitAbortedError()
 
 
+def assert_secure(repository, manifest):
+    sm = SecurityManager(repository)
+    sm.assert_secure(manifest, manifest.key)
+
+
+def recanonicalize_relative_location(cache_location, repository):
+    # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741).
+    repo_location = repository._location.canonical_path()
+    rl = Location(repo_location)
+    cl = Location(cache_location)
+    if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
+            and \
+            cl.path and rl.path and \
+            cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
+        # everything is same except the expected change in relative path canonicalization,
+        # update previous_location to avoid warning / user query about changed location:
+        return repo_location
+    else:
+        return cache_location
+
+
+def cache_dir(repository, path=None):
+    return path or os.path.join(get_cache_dir(), repository.id_str)
+
+
+class CacheConfig:
+    def __init__(self, repository, path=None, lock_wait=None):
+        self.repository = repository
+        self.path = cache_dir(repository, path)
+        self.config_path = os.path.join(self.path, 'config')
+        self.lock = None
+        self.lock_wait = lock_wait
+
+    def __enter__(self):
+        self.open()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+    def create(self):
+        assert not self.exists()
+        config = configparser.ConfigParser(interpolation=None)
+        config.add_section('cache')
+        config.set('cache', 'version', '1')
+        config.set('cache', 'repository', self.repository.id_str)
+        config.set('cache', 'manifest', '')
+        with SaveFile(self.config_path) as fd:
+            config.write(fd)
+
+    def open(self):
+        self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=self.lock_wait,
+                         kill_stale_locks=hostname_is_unique()).acquire()
+        self.load()
+
+    def load(self):
+        self._config = configparser.ConfigParser(interpolation=None)
+        self._config.read(self.config_path)
+        self._check_upgrade(self.config_path)
+        self.id = self._config.get('cache', 'repository')
+        self.manifest_id = unhexlify(self._config.get('cache', 'manifest'))
+        self.timestamp = self._config.get('cache', 'timestamp', fallback=None)
+        self.key_type = self._config.get('cache', 'key_type', fallback=None)
+        previous_location = self._config.get('cache', 'previous_location', fallback=None)
+        if previous_location:
+            self.previous_location = recanonicalize_relative_location(previous_location, self.repository)
+        else:
+            self.previous_location = None
+
+    def save(self, manifest=None, key=None):
+        if manifest:
+            self._config.set('cache', 'manifest', manifest.id_str)
+            self._config.set('cache', 'timestamp', manifest.timestamp)
+        if key:
+            self._config.set('cache', 'key_type', str(key.TYPE))
+        self._config.set('cache', 'previous_location', self.repository._location.canonical_path())
+        with SaveFile(self.config_path) as fd:
+            self._config.write(fd)
+
+    def close(self):
+        if self.lock is not None:
+            self.lock.release()
+            self.lock = None
+
+    def _check_upgrade(self, config_path):
+        try:
+            cache_version = self._config.getint('cache', 'version')
+            wanted_version = 1
+            if cache_version != wanted_version:
+                self.close()
+                raise Exception('%s has unexpected cache version %d (wanted: %d).' %
+                                (config_path, cache_version, wanted_version))
+        except configparser.NoSectionError:
+            self.close()
+            raise Exception('%s does not look like a Borg cache.' % config_path) from None
+
+
 class Cache:
     """Client Side cache
     """
@@ -158,7 +274,7 @@ class EncryptionMethodMismatch(Error):
 
     @staticmethod
     def break_lock(repository, path=None):
-        path = path or os.path.join(get_cache_dir(), repository.id_str)
+        path = cache_dir(repository, path)
         Lock(os.path.join(path, 'lock'), exclusive=True).break_lock()
 
     @staticmethod
@@ -178,25 +294,27 @@ def __init__(self, repository, key, manifest, path=None, sync=True, do_files=Fal
         :param lock_wait: timeout for lock acquisition (None: return immediately if lock unavailable)
         :param sync: do :meth:`.sync`
         """
-        self.lock = None
-        self.timestamp = None
-        self.lock = None
-        self.txn_active = False
         self.repository = repository
         self.key = key
         self.manifest = manifest
         self.progress = progress
-        self.path = path or os.path.join(get_cache_dir(), repository.id_str)
-        self.security_manager = SecurityManager(repository)
         self.do_files = do_files
+        self.timestamp = None
+        self.txn_active = False
+
+        self.path = cache_dir(repository, path)
+        self.security_manager = SecurityManager(repository)
+        self.cache_config = CacheConfig(self.repository, self.path, lock_wait)
+
         # Warn user before sending data to a never seen before unencrypted repository
         if not os.path.exists(self.path):
-            self.security_manager.assert_access_unknown(warn_if_unencrypted, key)
+            self.security_manager.assert_access_unknown(warn_if_unencrypted, manifest, key)
             self.create()
-        self.open(lock_wait=lock_wait)
+
+        self.open()
         try:
-            self.security_manager.assert_secure(manifest, key, self)
+            self.security_manager.assert_secure(manifest, key, cache_config=self.cache_config)
-            if sync and self.manifest.id != self.manifest_id:
+            if sync and self.manifest.id != self.cache_config.manifest_id:
                 self.sync()
                 self.commit()
         except:
@@ -240,66 +358,27 @@ def create(self):
         os.makedirs(self.path)
         with open(os.path.join(self.path, 'README'), 'w') as fd:
             fd.write(CACHE_README)
-        config = configparser.ConfigParser(interpolation=None)
+        self.cache_config.create()
-        config.add_section('cache')
-        config.set('cache', 'version', '1')
-        config.set('cache', 'repository', self.repository.id_str)
-        config.set('cache', 'manifest', '')
-        with SaveFile(os.path.join(self.path, 'config')) as fd:
-            config.write(fd)
         ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
         os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
             pass  # empty file
 
-    def _check_upgrade(self, config_path):
-        try:
-            cache_version = self.config.getint('cache', 'version')
-            wanted_version = 1
-            if cache_version != wanted_version:
-                self.close()
-                raise Exception('%s has unexpected cache version %d (wanted: %d).' % (
-                    config_path, cache_version, wanted_version))
-        except configparser.NoSectionError:
-            self.close()
-            raise Exception('%s does not look like a Borg cache.' % config_path) from None
-        # borg < 1.0.8rc1 had different canonicalization for the repo location (see #1655 and #1741).
-        cache_loc = self.config.get('cache', 'previous_location', fallback=None)
-        if cache_loc:
-            repo_loc = self.repository._location.canonical_path()
-            rl = Location(repo_loc)
-            cl = Location(cache_loc)
-            if cl.proto == rl.proto and cl.user == rl.user and cl.host == rl.host and cl.port == rl.port \
-                    and \
-                    cl.path and rl.path and \
-                    cl.path.startswith('/~/') and rl.path.startswith('/./') and cl.path[3:] == rl.path[3:]:
-                # everything is same except the expected change in relative path canonicalization,
-                # update previous_location to avoid warning / user query about changed location:
-                self.config.set('cache', 'previous_location', repo_loc)
-
     def _do_open(self):
-        self.config = configparser.ConfigParser(interpolation=None)
+        self.cache_config.load()
-        config_path = os.path.join(self.path, 'config')
-        self.config.read(config_path)
-        self._check_upgrade(config_path)
-        self.id = self.config.get('cache', 'repository')
-        self.manifest_id = unhexlify(self.config.get('cache', 'manifest'))
-        self.timestamp = self.config.get('cache', 'timestamp', fallback=None)
-        self.key_type = self.config.get('cache', 'key_type', fallback=None)
-        self.previous_location = self.config.get('cache', 'previous_location', fallback=None)
         self.chunks = ChunkIndex.read(os.path.join(self.path, 'chunks').encode('utf-8'))
         self.files = None
 
-    def open(self, lock_wait=None):
+    def open(self):
         if not os.path.isdir(self.path):
             raise Exception('%s Does not look like a Borg cache' % self.path)
-        self.lock = Lock(os.path.join(self.path, 'lock'), exclusive=True, timeout=lock_wait, kill_stale_locks=hostname_is_unique()).acquire()
+        self.cache_config.open()
         self.rollback()
 
     def close(self):
-        if self.lock is not None:
+        if self.cache_config is not None:
-            self.lock.release()
+            self.cache_config.close()
-            self.lock = None
+            self.cache_config = None
 
     def _read_files(self):
         self.files = {}
@@ -338,7 +417,7 @@ def commit(self):
         """
         if not self.txn_active:
             return
-        self.security_manager.save(self.manifest, self.key, self)
+        self.security_manager.save(self.manifest, self.key)
         pi = ProgressIndicatorMessage(msgid='cache.commit')
         if self.files is not None:
             if self._newest_mtime is None:
@@ -356,12 +435,7 @@ def commit(self):
                        entry.age > 0 and entry.age < ttl:
                         msgpack.pack((path_hash, entry), fd)
         pi.output('Saving cache config')
-        self.config.set('cache', 'manifest', self.manifest.id_str)
+        self.cache_config.save(self.manifest, self.key)
-        self.config.set('cache', 'timestamp', self.manifest.timestamp)
-        self.config.set('cache', 'key_type', str(self.key.TYPE))
-        self.config.set('cache', 'previous_location', self.repository._location.canonical_path())
-        with SaveFile(os.path.join(self.path, 'config')) as fd:
-            self.config.write(fd)
         pi.output('Saving chunks cache')
         self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8'))
         os.rename(os.path.join(self.path, 'txn.active'),