diff --git a/src/borg/cache.py b/src/borg/cache.py
index cb628a4e7..1550ae38c 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -644,14 +644,17 @@ class LocalCache(CacheStatsMixin):
 
             all_missing_ids = chunk_idx.zero_csize_ids()
             fetch_ids = []
-            for id_ in all_missing_ids:
-                already_fetched_entry = chunks_fetched_size_index.get(id_)
-                if already_fetched_entry:
-                    entry = chunk_idx[id_]._replace(csize=already_fetched_entry.csize)
-                    assert entry.size == already_fetched_entry.size, 'Chunk size mismatch'
-                    chunk_idx[id_] = entry
-                else:
-                    fetch_ids.append(id_)
+            if len(chunks_fetched_size_index):
+                for id_ in all_missing_ids:
+                    already_fetched_entry = chunks_fetched_size_index.get(id_)
+                    if already_fetched_entry:
+                        entry = chunk_idx[id_]._replace(csize=already_fetched_entry.csize)
+                        assert entry.size == already_fetched_entry.size, 'Chunk size mismatch'
+                        chunk_idx[id_] = entry
+                    else:
+                        fetch_ids.append(id_)
+            else:
+                fetch_ids = all_missing_ids
 
             for id_, data in zip(fetch_ids, decrypted_repository.repository.get_many(fetch_ids)):
                 entry = chunk_idx[id_]._replace(csize=len(data))
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 4e7cb5a58..b0b0a9b7f 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1041,6 +1041,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         create_stats = create_json['cache']['stats']
         info_stats = info_json['cache']['stats']
         assert create_stats == info_stats
+        self.cmd('delete', '--cache-only', self.repository_location)
+        self.cmd('create', '--no-cache-sync', self.repository_location + '::test2', 'input')
+        self.cmd('info', self.repository_location)
         self.cmd('check', self.repository_location)
 
     def test_extract_pattern_opt(self):