mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-01-02 21:25:26 +00:00
Code Issues Releases Wiki Activity

update ChunkIndex cache in repo every 10mins

Browse source
This commit is contained in:
Thomas Waldmann 2024-11-14 21:44:56 +01:00
parent d5d49e8a15
commit 295f4e3114
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
1 changed files with 10 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/borg/cache.py
View file

@ -833,6 +833,8 @@ def __init__(self):
self._chunks = None self._chunks = None
self.last_refresh_dt = datetime.now(timezone.utc) self.last_refresh_dt = datetime.now(timezone.utc)
self.refresh_td = timedelta(seconds=60) self.refresh_td = timedelta(seconds=60)
self.chunks_cache_last_write = datetime.now(timezone.utc)
self.chunks_cache_write_td = timedelta(seconds=600)
@property @property
def chunks(self): def chunks(self):
@ -879,6 +881,7 @@ def add_chunk(
else: else:
raise ValueError("when giving compressed data for a chunk, the uncompressed size must be given also") raise ValueError("when giving compressed data for a chunk, the uncompressed size must be given also")
now = datetime.now(timezone.utc) now = datetime.now(timezone.utc)
self._maybe_write_chunks_cache(now)
exists = self.seen_chunk(id, size) exists = self.seen_chunk(id, size)
if exists: if exists:
# if borg create is processing lots of unchanged files (no content and not metadata changes), # if borg create is processing lots of unchanged files (no content and not metadata changes),
@ -894,10 +897,10 @@ def add_chunk(
stats.update(size, not exists) stats.update(size, not exists)
return ChunkListEntry(id, size) return ChunkListEntry(id, size)
def _write_chunks_cache(self, chunks): def _maybe_write_chunks_cache(self, now, force=False, clear=False):
# this is called from .close, so we can clear here: if force or now > self.chunks_cache_last_write + self.chunks_cache_write_td:
write_chunkindex_to_repo_cache(self.repository, self._chunks, clear=True) write_chunkindex_to_repo_cache(self.repository, self._chunks, clear=clear)
self._chunks = None # nothing there (cleared!) self.chunks_cache_last_write = now
def refresh_lock(self, now): def refresh_lock(self, now):
if now > self.last_refresh_dt + self.refresh_td: if now > self.last_refresh_dt + self.refresh_td:
@ -995,7 +998,9 @@ def close(self):
for key, value in sorted(self._chunks.stats.items()): for key, value in sorted(self._chunks.stats.items()):
logger.debug(f"Chunks index stats: {key}: {value}") logger.debug(f"Chunks index stats: {key}: {value}")
pi.output("Saving chunks cache") pi.output("Saving chunks cache")
self._write_chunks_cache(self._chunks) # cache/chunks in repo has a different integrity mechanism # note: cache/chunks.* in repo has a different integrity mechanism
self._maybe_write_chunks_cache(self._chunks, force=True, clear=True)
self._chunks = None # nothing there (cleared!)
pi.output("Saving cache config") pi.output("Saving cache config")
self.cache_config.save(self.manifest) self.cache_config.save(self.manifest)
self.cache_config.close() self.cache_config.close()