From 6ec6fc2496bbb77bd5dac90cb25060997e46376b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 13 Sep 2022 14:08:03 +0200 Subject: [PATCH] move corruption tests to own module --- src/borg/testsuite/archiver/__init__.py | 93 +------------------- src/borg/testsuite/archiver/corruption.py | 102 ++++++++++++++++++++++ 2 files changed, 103 insertions(+), 92 deletions(-) create mode 100644 src/borg/testsuite/archiver/corruption.py diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 06b933ad8..a18ec1944 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -1,6 +1,5 @@ import errno import io -import json import os import shutil import stat @@ -21,7 +20,6 @@ from ...archiver import Archiver, PURE_PYTHON_MSGPACK_WARNING from ...archiver._common import build_filter from ...cache import Cache from ...constants import * # NOQA -from ...crypto.file_integrity import FileIntegrityError from ...helpers import Location from ...helpers import EXIT_SUCCESS from ...helpers import bin_to_hex @@ -32,7 +30,7 @@ from ...logger import setup_logging from ...remote import RemoteRepository from ...repository import Repository from .. import has_lchflags -from .. import BaseTestCase, changedir +from .. import BaseTestCase, changedir, environment_variable from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported RK_ENCRYPTION = "--encryption=repokey-aes-ocb" @@ -360,21 +358,6 @@ class ArchiverTestCaseBase(BaseTestCase): class ArchiverTestCase(ArchiverTestCaseBase): - def test_corrupted_repository(self): - self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - self.create_src_archive("test") - self.cmd(f"--repo={self.repository_location}", "extract", "test", "--dry-run") - output = self.cmd(f"--repo={self.repository_location}", "check", "--show-version") - self.assert_in("borgbackup version", output) # implied output even without --info given - self.assert_not_in("Starting repository check", output) # --info not given for root logger - - name = sorted(os.listdir(os.path.join(self.tmpdir, "repository", "data", "0")), reverse=True)[1] - with open(os.path.join(self.tmpdir, "repository", "data", "0", name), "r+b") as fd: - fd.seek(100) - fd.write(b"XXXX") - output = self.cmd(f"--repo={self.repository_location}", "check", "--info", exit_code=1) - self.assert_in("Starting repository check", output) # --info given for root logger - def test_bad_filters(self): self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.cmd(f"--repo={self.repository_location}", "create", "test", "input") @@ -411,80 +394,6 @@ class RemoteArchiverTestCaseBase: return RemoteRepository(Location(self.repository_location)) -class ArchiverCorruptionTestCase(ArchiverTestCaseBase): - def setUp(self): - super().setUp() - self.create_test_files() - self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - self.cache_path = json.loads(self.cmd(f"--repo={self.repository_location}", "rinfo", "--json"))["cache"]["path"] - - def corrupt(self, file, amount=1): - with open(file, "r+b") as fd: - fd.seek(-amount, io.SEEK_END) - corrupted = bytes(255 - c for c in fd.read(amount)) - fd.seek(-amount, io.SEEK_END) - fd.write(corrupted) - - def test_cache_chunks(self): - self.corrupt(os.path.join(self.cache_path, "chunks")) - - if self.FORK_DEFAULT: - out = self.cmd(f"--repo={self.repository_location}", "rinfo", exit_code=2) - assert "failed integrity check" in out - else: - with pytest.raises(FileIntegrityError): - self.cmd(f"--repo={self.repository_location}", "rinfo") - - def test_cache_files(self): - self.cmd(f"--repo={self.repository_location}", "create", "test", "input") - self.corrupt(os.path.join(self.cache_path, "files")) - out = self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") - # borg warns about the corrupt files cache, but then continues without files cache. - assert "files cache is corrupted" in out - - def test_chunks_archive(self): - self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") - # Find ID of test1 so we can corrupt it later :) - target_id = self.cmd(f"--repo={self.repository_location}", "rlist", "--format={id}{LF}").strip() - self.cmd(f"--repo={self.repository_location}", "create", "test2", "input") - - # Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d - self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") - self.cmd(f"--repo={self.repository_location}", "rinfo", "--json") - - chunks_archive = os.path.join(self.cache_path, "chunks.archive.d") - assert len(os.listdir(chunks_archive)) == 4 # two archives, one chunks cache and one .integrity file each - - self.corrupt(os.path.join(chunks_archive, target_id + ".compact")) - - # Trigger cache sync by changing the manifest ID in the cache config - config_path = os.path.join(self.cache_path, "config") - config = ConfigParser(interpolation=None) - config.read(config_path) - config.set("cache", "manifest", bin_to_hex(bytes(32))) - with open(config_path, "w") as fd: - config.write(fd) - - # Cache sync notices corrupted archive chunks, but automatically recovers. - out = self.cmd(f"--repo={self.repository_location}", "create", "-v", "test3", "input", exit_code=1) - assert "Reading cached archive chunk index for test1" in out - assert "Cached archive chunk index of test1 is corrupted" in out - assert "Fetching and building archive index for test1" in out - - def test_old_version_interfered(self): - # Modify the main manifest ID without touching the manifest ID in the integrity section. - # This happens if a version without integrity checking modifies the cache. - config_path = os.path.join(self.cache_path, "config") - config = ConfigParser(interpolation=None) - config.read(config_path) - config.set("cache", "manifest", bin_to_hex(bytes(32))) - with open(config_path, "w") as fd: - config.write(fd) - - out = self.cmd(f"--repo={self.repository_location}", "rinfo") - assert "Cache integrity data not available: old Borg version modified the cache." in out - - class TestBuildFilter: def test_basic(self): matcher = PatternMatcher() diff --git a/src/borg/testsuite/archiver/corruption.py b/src/borg/testsuite/archiver/corruption.py new file mode 100644 index 000000000..1d490d921 --- /dev/null +++ b/src/borg/testsuite/archiver/corruption.py @@ -0,0 +1,102 @@ +import io +import json +import os +from configparser import ConfigParser + +import pytest + +from ...constants import * # NOQA +from ...crypto.file_integrity import FileIntegrityError +from ...helpers import bin_to_hex +from . import ArchiverTestCaseBase, RK_ENCRYPTION + + +class ArchiverTestCase(ArchiverTestCaseBase): + def test_corrupted_repository(self): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.create_src_archive("test") + self.cmd(f"--repo={self.repository_location}", "extract", "test", "--dry-run") + output = self.cmd(f"--repo={self.repository_location}", "check", "--show-version") + self.assert_in("borgbackup version", output) # implied output even without --info given + self.assert_not_in("Starting repository check", output) # --info not given for root logger + + name = sorted(os.listdir(os.path.join(self.tmpdir, "repository", "data", "0")), reverse=True)[1] + with open(os.path.join(self.tmpdir, "repository", "data", "0", name), "r+b") as fd: + fd.seek(100) + fd.write(b"XXXX") + output = self.cmd(f"--repo={self.repository_location}", "check", "--info", exit_code=1) + self.assert_in("Starting repository check", output) # --info given for root logger + + +class ArchiverCorruptionTestCase(ArchiverTestCaseBase): + def setUp(self): + super().setUp() + self.create_test_files() + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + self.cache_path = json.loads(self.cmd(f"--repo={self.repository_location}", "rinfo", "--json"))["cache"]["path"] + + def corrupt(self, file, amount=1): + with open(file, "r+b") as fd: + fd.seek(-amount, io.SEEK_END) + corrupted = bytes(255 - c for c in fd.read(amount)) + fd.seek(-amount, io.SEEK_END) + fd.write(corrupted) + + def test_cache_chunks(self): + self.corrupt(os.path.join(self.cache_path, "chunks")) + + if self.FORK_DEFAULT: + out = self.cmd(f"--repo={self.repository_location}", "rinfo", exit_code=2) + assert "failed integrity check" in out + else: + with pytest.raises(FileIntegrityError): + self.cmd(f"--repo={self.repository_location}", "rinfo") + + def test_cache_files(self): + self.cmd(f"--repo={self.repository_location}", "create", "test", "input") + self.corrupt(os.path.join(self.cache_path, "files")) + out = self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") + # borg warns about the corrupt files cache, but then continues without files cache. + assert "files cache is corrupted" in out + + def test_chunks_archive(self): + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") + # Find ID of test1 so we can corrupt it later :) + target_id = self.cmd(f"--repo={self.repository_location}", "rlist", "--format={id}{LF}").strip() + self.cmd(f"--repo={self.repository_location}", "create", "test2", "input") + + # Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d + self.cmd(f"--repo={self.repository_location}", "rdelete", "--cache-only") + self.cmd(f"--repo={self.repository_location}", "rinfo", "--json") + + chunks_archive = os.path.join(self.cache_path, "chunks.archive.d") + assert len(os.listdir(chunks_archive)) == 4 # two archives, one chunks cache and one .integrity file each + + self.corrupt(os.path.join(chunks_archive, target_id + ".compact")) + + # Trigger cache sync by changing the manifest ID in the cache config + config_path = os.path.join(self.cache_path, "config") + config = ConfigParser(interpolation=None) + config.read(config_path) + config.set("cache", "manifest", bin_to_hex(bytes(32))) + with open(config_path, "w") as fd: + config.write(fd) + + # Cache sync notices corrupted archive chunks, but automatically recovers. + out = self.cmd(f"--repo={self.repository_location}", "create", "-v", "test3", "input", exit_code=1) + assert "Reading cached archive chunk index for test1" in out + assert "Cached archive chunk index of test1 is corrupted" in out + assert "Fetching and building archive index for test1" in out + + def test_old_version_interfered(self): + # Modify the main manifest ID without touching the manifest ID in the integrity section. + # This happens if a version without integrity checking modifies the cache. + config_path = os.path.join(self.cache_path, "config") + config = ConfigParser(interpolation=None) + config.read(config_path) + config.set("cache", "manifest", bin_to_hex(bytes(32))) + with open(config_path, "w") as fd: + config.write(fd) + + out = self.cmd(f"--repo={self.repository_location}", "rinfo") + assert "Cache integrity data not available: old Borg version modified the cache." in out