repository3/manifest: tests reenabled, fixes

src/borg/archive.py

@@ -52,7 +52,7 @@
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth, hostname
 from .remote import cache_if_remote
 from .remote3 import RemoteRepository3
-from .repository3 import Repository3, LIST_SCAN_LIMIT
+from .repository3 import Repository3, LIST_SCAN_LIMIT, NoManifestError
 from .repoobj import RepoObj
 
 has_link = hasattr(os, "link")
@@ -1860,8 +1860,9 @@ def check(
         self.repo_objs = RepoObj(self.key)
         if verify_data:
             self.verify_data()
-        if not isinstance(repository, (Repository3, RemoteRepository3)) and Manifest.MANIFEST_ID not in self.chunks:
+        try:
-            logger.error("Repository manifest not found!")
+            repository.get_manifest()
+        except NoManifestError:
             self.error_found = True
             self.manifest = self.rebuild_manifest()
         else:
@@ -1905,12 +1906,16 @@ def make_key(self, repository):
 
         #  try the manifest first!
         attempt += 1
-        cdata = repository.get_manifest()
         try:
-            return key_factory(repository, cdata)
+            cdata = repository.get_manifest()
-        except UnsupportedPayloadError:
+        except NoManifestError:
-            # we get here, if the cdata we got has a corrupted key type byte
+            pass
-            pass  # ignore it, just continue trying
+        else:
+            try:
+                return key_factory(repository, cdata)
+            except UnsupportedPayloadError:
+                # we get here, if the cdata we got has a corrupted key type byte
+                pass  # ignore it, just continue trying
 
         for chunkid, _ in self.chunks.iteritems():
             attempt += 1

src/borg/remote3.py

@@ -32,6 +32,7 @@
 from .helpers import get_socket_filename
 from .locking import LockTimeout, NotLocked, NotMyLock, LockFailed
 from .logger import create_logger, borg_serve_log_queue
+from .manifest import NoManifestError
 from .helpers import msgpack
 from .repository import Repository
 from .repository3 import Repository3
@@ -835,6 +836,8 @@ def handle_error(unpacked):
                 raise NotLocked(args[0])
             elif error == "NotMyLock":
                 raise NotMyLock(args[0])
+            elif error == "NoManifestError":
+                raise NoManifestError
             else:
                 raise self.RPCError(unpacked)
 

src/borg/testsuite/archiver/check_cmd.py

@@ -8,6 +8,7 @@
 from ...constants import *  # NOQA
 from ...helpers import bin_to_hex, msgpack
 from ...manifest import Manifest
+from ...remote3 import RemoteRepository3
 from ...repository3 import Repository3
 from ..repository3 import fchunk
 from . import cmd, src_file, create_src_archive, open_archive, generate_archiver_tests, RK_ENCRYPTION
@@ -192,11 +193,12 @@ def test_missing_manifest(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     with repository:
-        repository.delete(Manifest.MANIFEST_ID)
+        if isinstance(repository, (Repository3, RemoteRepository3)):
-        repository.commit(compact=False)
+            repository.store_delete("config/manifest")
+        else:
+            repository.delete(Manifest.MANIFEST_ID)
+            repository.commit(compact=False)
     cmd(archiver, "check", exit_code=1)
     output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
     assert "archive1" in output
@@ -208,12 +210,10 @@ def test_corrupted_manifest(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     with repository:
-        manifest = repository.get(Manifest.MANIFEST_ID)
+        manifest = repository.get_manifest()
         corrupted_manifest = manifest[:123] + b"corrupted!" + manifest[123:]
-        repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+        repository.put_manifest(corrupted_manifest)
         repository.commit(compact=False)
     cmd(archiver, "check", exit_code=1)
     output = cmd(archiver, "check", "-v", "--repair", exit_code=0)
@@ -226,8 +226,6 @@ def test_spoofed_manifest(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     with repository:
         manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
         cdata = manifest.repo_objs.format(
@@ -247,7 +245,7 @@ def test_spoofed_manifest(archivers, request):
         )
         # maybe a repo-side attacker could manage to move the fake manifest file chunk over to the manifest ID.
         # we simulate this here by directly writing the fake manifest data to the manifest ID.
-        repository.put(Manifest.MANIFEST_ID, cdata)
+        repository.put_manifest(cdata)
         repository.commit(compact=False)
     # borg should notice that the manifest has the wrong ro_type.
     cmd(archiver, "check", exit_code=1)
@@ -262,12 +260,10 @@ def test_manifest_rebuild_corrupted_chunk(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     with repository:
-        manifest = repository.get(Manifest.MANIFEST_ID)
+        manifest = repository.get_manifest()
         corrupted_manifest = manifest[:123] + b"corrupted!" + manifest[123:]
-        repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+        repository.put_manifest(corrupted_manifest)
         chunk = repository.get(archive.id)
         corrupted_chunk = chunk + b"corrupted!"
         repository.put(archive.id, corrupted_chunk)
@@ -282,13 +278,11 @@ def test_manifest_rebuild_duplicate_archive(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     repo_objs = archive.repo_objs
     with repository:
-        manifest = repository.get(Manifest.MANIFEST_ID)
+        manifest = repository.get_manifest()
         corrupted_manifest = manifest[:123] + b"corrupted!" + manifest[123:]
-        repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+        repository.put_manifest(corrupted_manifest)
         archive_dict = {
             "command_line": "",
             "item_ptrs": [],
@@ -314,14 +308,12 @@ def test_spoofed_archive(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)
     archive, repository = open_archive(archiver.repository_path, "archive1")
-    if isinstance(repository, Repository3):
-        pytest.skip("Test not adapted to Repository3")
     repo_objs = archive.repo_objs
     with repository:
         # attacker would corrupt or delete the manifest to trigger a rebuild of it:
-        manifest = repository.get(Manifest.MANIFEST_ID)
+        manifest = repository.get_manifest()
         corrupted_manifest = manifest[:123] + b"corrupted!" + manifest[123:]
-        repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
+        repository.put_manifest(corrupted_manifest)
         archive_dict = {
             "command_line": "",
             "item_ptrs": [],