mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-31 12:17:42 +00:00
Code Issues Releases Wiki Activity

when building the chunk index, merge all we have in the cache

Browse source
This commit is contained in:
Thomas Waldmann 2024-11-08 20:24:33 +01:00
parent 4a6fcc26d7
commit 00f8cdc9a7
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
1 changed files with 36 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
src/borg/cache.py
View file

@ -739,13 +739,7 @@ def write_chunkindex_to_repo_cache(repository, chunks, *, clear=False, force_wri
return new_hash return new_hash
def build_chunkindex_from_repo(repository, *, disable_caches=False, cache_immediately=False): def read_chunkindex_from_repo_cache(repository, hash):
try_upgrade_to_b14(repository)
# first, try to load a pre-computed and centrally cached chunks index:
if not disable_caches:
hashes = list_chunkindex_hashes(repository)
assert len(hashes) <= 1, f"chunk indexes: {hashes}" # later we change to multiple chunkindexes...
for hash in hashes:
cache_name = f"cache/chunks.{hash}" cache_name = f"cache/chunks.{hash}"
logger.debug(f"trying to load {cache_name} from the repo...") logger.debug(f"trying to load {cache_name} from the repo...")
try: try:
@ -761,6 +755,29 @@ def build_chunkindex_from_repo(repository, *, disable_caches=False, cache_immedi
return chunks return chunks
else: else:
logger.debug(f"{cache_name} is invalid.") logger.debug(f"{cache_name} is invalid.")
def build_chunkindex_from_repo(repository, *, disable_caches=False, cache_immediately=False):
try_upgrade_to_b14(repository)
# first, try to build a fresh, mostly complete chunk index from centrally cached chunk indexes:
if not disable_caches:
hashes = list_chunkindex_hashes(repository)
if hashes: # we have at least one cached chunk index!
merged = 0
chunks = ChunkIndex() # we'll merge all we find into this
for hash in hashes:
chunks_to_merge = read_chunkindex_from_repo_cache(repository, hash)
if chunks_to_merge is not None:
logger.debug(f"cached chunk index {hash} gets merged...")
for k, v in chunks_to_merge.items():
chunks[k] = v
merged += 1
chunks_to_merge.clear()
if merged > 0:
if merged > 1 and cache_immediately:
# immediately update cache/chunks, so we don't have to merge these again:
write_chunkindex_to_repo_cache(repository, chunks, clear=False, force_write=True, delete_other=True)
return chunks
# if we didn't get anything from the cache, compute the ChunkIndex the slow way: # if we didn't get anything from the cache, compute the ChunkIndex the slow way:
logger.debug("querying the chunk IDs list from the repo...") logger.debug("querying the chunk IDs list from the repo...")
chunks = ChunkIndex() chunks = ChunkIndex()
@ -858,7 +875,7 @@ def add_chunk(
def _write_chunks_cache(self, chunks): def _write_chunks_cache(self, chunks):
# this is called from .close, so we can clear here: # this is called from .close, so we can clear here:
write_chunkindex_to_repo_cache(self.repository, self._chunks, clear=True, delete_other=True) write_chunkindex_to_repo_cache(self.repository, self._chunks, clear=True, delete_other=False)
self._chunks = None # nothing there (cleared!) self._chunks = None # nothing there (cleared!)
def refresh_lock(self, now): def refresh_lock(self, now):