diff --git a/src/borg/cache.py b/src/borg/cache.py index 13045f0e9..344682a94 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -15,8 +15,9 @@ from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Location from .helpers import Error +from .helpers import Manifest from .helpers import get_cache_dir, get_security_dir -from .helpers import int_to_bigint, bigint_to_int, bin_to_hex +from .helpers import int_to_bigint, bigint_to_int, bin_to_hex, parse_stringified_list from .helpers import format_file_size from .helpers import safe_ns from .helpers import yes, hostname_is_unique @@ -257,6 +258,8 @@ def load(self): self.manifest_id = unhexlify(self._config.get('cache', 'manifest')) self.timestamp = self._config.get('cache', 'timestamp', fallback=None) self.key_type = self._config.get('cache', 'key_type', fallback=None) + self.ignored_features = set(parse_stringified_list(self._config.get('cache', 'ignored_features', fallback=''))) + self.mandatory_features = set(parse_stringified_list(self._config.get('cache', 'mandatory_features', fallback=''))) try: self.integrity = dict(self._config.items('integrity')) if self._config.get('cache', 'manifest') != self.integrity.pop('manifest'): @@ -281,6 +284,8 @@ def save(self, manifest=None, key=None): if manifest: self._config.set('cache', 'manifest', manifest.id_str) self._config.set('cache', 'timestamp', manifest.timestamp) + self._config.set('cache', 'ignored_features', ','.join(self.ignored_features)) + self._config.set('cache', 'mandatory_features', ','.join(self.mandatory_features)) if not self._config.has_section('integrity'): self._config.add_section('integrity') for file, integrity_data in self.integrity.items(): @@ -370,6 +375,12 @@ def __init__(self, repository, key, manifest, path=None, sync=True, do_files=Fal self.open() try: self.security_manager.assert_secure(manifest, key, cache_config=self.cache_config) + + if not self.check_cache_compatibility(): + self.wipe_cache() + + self.update_compatibility() + if sync and self.manifest.id != self.cache_config.manifest_id: self.sync() self.commit() @@ -678,6 +689,43 @@ def legacy_cleanup(): self.do_cache = os.path.isdir(archive_path) self.chunks = create_master_idx(self.chunks) + def check_cache_compatibility(self): + my_features = Manifest.SUPPORTED_REPO_FEATURES + if self.cache_config.ignored_features & my_features: + # The cache might not contain references of chunks that need a feature that is mandatory for some operation + # and which this version supports. To avoid corruption while executing that operation force rebuild. + return False + if not self.cache_config.mandatory_features <= my_features: + # The cache was build with consideration to at least one feature that this version does not understand. + # This client might misinterpret the cache. Thus force a rebuild. + return False + return True + + def wipe_cache(self): + logger.warning("Discarding incompatible cache and forcing a cache rebuild") + archive_path = os.path.join(self.path, 'chunks.archive.d') + if os.path.isdir(archive_path): + shutil.rmtree(os.path.join(self.path, 'chunks.archive.d')) + os.makedirs(os.path.join(self.path, 'chunks.archive.d')) + self.chunks = ChunkIndex() + with open(os.path.join(self.path, 'files'), 'wb'): + pass # empty file + self.cache_config.manifest_id = '' + self.cache_config._config.set('cache', 'manifest', '') + + self.cache_config.ignored_features = set() + self.cache_config.mandatory_features = set() + + def update_compatibility(self): + operation_to_features_map = self.manifest.get_all_mandatory_features() + my_features = Manifest.SUPPORTED_REPO_FEATURES + repo_features = set() + for operation, features in operation_to_features_map.items(): + repo_features.update(features) + + self.cache_config.ignored_features.update(repo_features - my_features) + self.cache_config.mandatory_features.update(repo_features & my_features) + def add_chunk(self, id, chunk, stats, overwrite=False, wait=True): if not self.txn_active: self.begin_txn() diff --git a/src/borg/helpers.py b/src/borg/helpers.py index d0246bddd..f14f06441 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -325,6 +325,17 @@ def check_repository_compatibility(self, operations): if unsupported: raise MandatoryFeatureUnsupported([f.decode() for f in unsupported]) + def get_all_mandatory_features(self): + result = {} + feature_flags = self.config.get(b'feature_flags', None) + if feature_flags is None: + return result + + for operation, requirements in feature_flags.items(): + if b'mandatory' in requirements: + result[operation.decode()] = set([feature.decode() for feature in requirements[b'mandatory']]) + return result + def write(self): from .item import ManifestItem if self.key.tam_required: @@ -823,6 +834,11 @@ def bin_to_hex(binary): return hexlify(binary).decode('ascii') +def parse_stringified_list(s): + l = re.split(" *, *", s) + return [item for item in l if item != ''] + + class Location: """Object representing a repository / archive location """