From 185951fb57ae92d1390451da52915b68c89c924b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 12 Sep 2022 01:42:36 +0200 Subject: [PATCH] move diff cmd tests to own module --- src/borg/testsuite/archiver/__init__.py | 249 +---------------------- src/borg/testsuite/archiver/diff_cmd.py | 253 ++++++++++++++++++++++++ 2 files changed, 254 insertions(+), 248 deletions(-) create mode 100644 src/borg/testsuite/archiver/diff_cmd.py diff --git a/src/borg/testsuite/archiver/__init__.py b/src/borg/testsuite/archiver/__init__.py index 1d2492086..9257eda00 100644 --- a/src/borg/testsuite/archiver/__init__.py +++ b/src/borg/testsuite/archiver/__init__.py @@ -2,7 +2,6 @@ import errno import io import json -import logging import os import random import re @@ -25,7 +24,7 @@ import borg import borg.helpers.errors from ... import xattr, helpers, platform -from ...archive import Archive, ChunkBuffer +from ...archive import Archive from ...archiver import Archiver, PURE_PYTHON_MSGPACK_WARNING from ...archiver._common import build_filter from ...cache import Cache, LocalCache @@ -3708,252 +3707,6 @@ def test_old_version_interfered(self): assert "Cache integrity data not available: old Borg version modified the cache." in out -class DiffArchiverTestCase(ArchiverTestCaseBase): - def test_basic_functionality(self): - # Setup files for the first snapshot - self.create_regular_file("empty", size=0) - self.create_regular_file("file_unchanged", size=128) - self.create_regular_file("file_removed", size=256) - self.create_regular_file("file_removed2", size=512) - self.create_regular_file("file_replaced", size=1024) - os.mkdir("input/dir_replaced_with_file") - os.chmod("input/dir_replaced_with_file", stat.S_IFDIR | 0o755) - os.mkdir("input/dir_removed") - if are_symlinks_supported(): - os.mkdir("input/dir_replaced_with_link") - os.symlink("input/dir_replaced_with_file", "input/link_changed") - os.symlink("input/file_unchanged", "input/link_removed") - os.symlink("input/file_removed2", "input/link_target_removed") - os.symlink("input/empty", "input/link_target_contents_changed") - os.symlink("input/empty", "input/link_replaced_by_file") - if are_hardlinks_supported(): - os.link("input/file_replaced", "input/hardlink_target_replaced") - os.link("input/empty", "input/hardlink_contents_changed") - os.link("input/file_removed", "input/hardlink_removed") - os.link("input/file_removed2", "input/hardlink_target_removed") - - self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - - # Create the first snapshot - self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") - - # Setup files for the second snapshot - self.create_regular_file("file_added", size=2048) - self.create_regular_file("file_empty_added", size=0) - os.unlink("input/file_replaced") - self.create_regular_file("file_replaced", contents=b"0" * 4096) - os.unlink("input/file_removed") - os.unlink("input/file_removed2") - os.rmdir("input/dir_replaced_with_file") - self.create_regular_file("dir_replaced_with_file", size=8192) - os.chmod("input/dir_replaced_with_file", stat.S_IFREG | 0o755) - os.mkdir("input/dir_added") - os.rmdir("input/dir_removed") - if are_symlinks_supported(): - os.rmdir("input/dir_replaced_with_link") - os.symlink("input/dir_added", "input/dir_replaced_with_link") - os.unlink("input/link_changed") - os.symlink("input/dir_added", "input/link_changed") - os.symlink("input/dir_added", "input/link_added") - os.unlink("input/link_replaced_by_file") - self.create_regular_file("link_replaced_by_file", size=16384) - os.unlink("input/link_removed") - if are_hardlinks_supported(): - os.unlink("input/hardlink_removed") - os.link("input/file_added", "input/hardlink_added") - - with open("input/empty", "ab") as fd: - fd.write(b"appended_data") - - # Create the second snapshot - self.cmd(f"--repo={self.repository_location}", "create", "test1a", "input") - self.cmd(f"--repo={self.repository_location}", "create", "test1b", "input", "--chunker-params", "16,18,17,4095") - - def do_asserts(output, can_compare_ids): - # File contents changed (deleted and replaced with a new file) - change = "B" if can_compare_ids else "{:<19}".format("modified") - assert "file_replaced" in output # added to debug #3494 - assert f"{change} input/file_replaced" in output - - # File unchanged - assert "input/file_unchanged" not in output - - # Directory replaced with a regular file - if "BORG_TESTS_IGNORE_MODES" not in os.environ: - assert "[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file" in output - - # Basic directory cases - assert "added directory input/dir_added" in output - assert "removed directory input/dir_removed" in output - - if are_symlinks_supported(): - # Basic symlink cases - assert "changed link input/link_changed" in output - assert "added link input/link_added" in output - assert "removed link input/link_removed" in output - - # Symlink replacing or being replaced - assert "] input/dir_replaced_with_link" in output - assert "] input/link_replaced_by_file" in output - - # Symlink target removed. Should not affect the symlink at all. - assert "input/link_target_removed" not in output - - # The inode has two links and the file contents changed. Borg - # should notice the changes in both links. However, the symlink - # pointing to the file is not changed. - change = "0 B" if can_compare_ids else "{:<19}".format("modified") - assert f"{change} input/empty" in output - if are_hardlinks_supported(): - assert f"{change} input/hardlink_contents_changed" in output - if are_symlinks_supported(): - assert "input/link_target_contents_changed" not in output - - # Added a new file and a hard link to it. Both links to the same - # inode should appear as separate files. - assert "added 2.05 kB input/file_added" in output - if are_hardlinks_supported(): - assert "added 2.05 kB input/hardlink_added" in output - - # check if a diff between non-existent and empty new file is found - assert "added 0 B input/file_empty_added" in output - - # The inode has two links and both of them are deleted. They should - # appear as two deleted files. - assert "removed 256 B input/file_removed" in output - if are_hardlinks_supported(): - assert "removed 256 B input/hardlink_removed" in output - - # Another link (marked previously as the source in borg) to the - # same inode was removed. This should not change this link at all. - if are_hardlinks_supported(): - assert "input/hardlink_target_removed" not in output - - # Another link (marked previously as the source in borg) to the - # same inode was replaced with a new regular file. This should not - # change this link at all. - if are_hardlinks_supported(): - assert "input/hardlink_target_replaced" not in output - - def do_json_asserts(output, can_compare_ids): - def get_changes(filename, data): - chgsets = [j["changes"] for j in data if j["path"] == filename] - assert len(chgsets) < 2 - # return a flattened list of changes for given filename - return [chg for chgset in chgsets for chg in chgset] - - # convert output to list of dicts - joutput = [json.loads(line) for line in output.split("\n") if line] - - # File contents changed (deleted and replaced with a new file) - expected = {"type": "modified", "added": 4096, "removed": 1024} if can_compare_ids else {"type": "modified"} - assert expected in get_changes("input/file_replaced", joutput) - - # File unchanged - assert not any(get_changes("input/file_unchanged", joutput)) - - # Directory replaced with a regular file - if "BORG_TESTS_IGNORE_MODES" not in os.environ: - assert {"type": "mode", "old_mode": "drwxr-xr-x", "new_mode": "-rwxr-xr-x"} in get_changes( - "input/dir_replaced_with_file", joutput - ) - - # Basic directory cases - assert {"type": "added directory"} in get_changes("input/dir_added", joutput) - assert {"type": "removed directory"} in get_changes("input/dir_removed", joutput) - - if are_symlinks_supported(): - # Basic symlink cases - assert {"type": "changed link"} in get_changes("input/link_changed", joutput) - assert {"type": "added link"} in get_changes("input/link_added", joutput) - assert {"type": "removed link"} in get_changes("input/link_removed", joutput) - - # Symlink replacing or being replaced - assert any( - chg["type"] == "mode" and chg["new_mode"].startswith("l") - for chg in get_changes("input/dir_replaced_with_link", joutput) - ) - assert any( - chg["type"] == "mode" and chg["old_mode"].startswith("l") - for chg in get_changes("input/link_replaced_by_file", joutput) - ) - - # Symlink target removed. Should not affect the symlink at all. - assert not any(get_changes("input/link_target_removed", joutput)) - - # The inode has two links and the file contents changed. Borg - # should notice the changes in both links. However, the symlink - # pointing to the file is not changed. - expected = {"type": "modified", "added": 13, "removed": 0} if can_compare_ids else {"type": "modified"} - assert expected in get_changes("input/empty", joutput) - if are_hardlinks_supported(): - assert expected in get_changes("input/hardlink_contents_changed", joutput) - if are_symlinks_supported(): - assert not any(get_changes("input/link_target_contents_changed", joutput)) - - # Added a new file and a hard link to it. Both links to the same - # inode should appear as separate files. - assert {"type": "added", "size": 2048} in get_changes("input/file_added", joutput) - if are_hardlinks_supported(): - assert {"type": "added", "size": 2048} in get_changes("input/hardlink_added", joutput) - - # check if a diff between non-existent and empty new file is found - assert {"type": "added", "size": 0} in get_changes("input/file_empty_added", joutput) - - # The inode has two links and both of them are deleted. They should - # appear as two deleted files. - assert {"type": "removed", "size": 256} in get_changes("input/file_removed", joutput) - if are_hardlinks_supported(): - assert {"type": "removed", "size": 256} in get_changes("input/hardlink_removed", joutput) - - # Another link (marked previously as the source in borg) to the - # same inode was removed. This should not change this link at all. - if are_hardlinks_supported(): - assert not any(get_changes("input/hardlink_target_removed", joutput)) - - # Another link (marked previously as the source in borg) to the - # same inode was replaced with a new regular file. This should not - # change this link at all. - if are_hardlinks_supported(): - assert not any(get_changes("input/hardlink_target_replaced", joutput)) - - do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a"), True) - # We expect exit_code=1 due to the chunker params warning - do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1b", exit_code=1), False) - do_json_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a", "--json-lines"), True) - - def test_sort_option(self): - self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) - - self.create_regular_file("a_file_removed", size=8) - self.create_regular_file("f_file_removed", size=16) - self.create_regular_file("c_file_changed", size=32) - self.create_regular_file("e_file_changed", size=64) - self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") - - os.unlink("input/a_file_removed") - os.unlink("input/f_file_removed") - os.unlink("input/c_file_changed") - os.unlink("input/e_file_changed") - self.create_regular_file("c_file_changed", size=512) - self.create_regular_file("e_file_changed", size=1024) - self.create_regular_file("b_file_added", size=128) - self.create_regular_file("d_file_added", size=256) - self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") - - output = self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1", "--sort") - expected = [ - "a_file_removed", - "b_file_added", - "c_file_changed", - "d_file_added", - "e_file_changed", - "f_file_removed", - ] - - assert all(x in line for x, line in zip(expected, output.splitlines())) - - def test_get_args(): archiver = Archiver() # everything normal: diff --git a/src/borg/testsuite/archiver/diff_cmd.py b/src/borg/testsuite/archiver/diff_cmd.py new file mode 100644 index 000000000..43aa7a3ac --- /dev/null +++ b/src/borg/testsuite/archiver/diff_cmd.py @@ -0,0 +1,253 @@ +import json +import os +import stat + +from ...constants import * # NOQA +from .. import are_symlinks_supported, are_hardlinks_supported +from . import ArchiverTestCaseBase, RK_ENCRYPTION + + +class DiffArchiverTestCase(ArchiverTestCaseBase): + def test_basic_functionality(self): + # Setup files for the first snapshot + self.create_regular_file("empty", size=0) + self.create_regular_file("file_unchanged", size=128) + self.create_regular_file("file_removed", size=256) + self.create_regular_file("file_removed2", size=512) + self.create_regular_file("file_replaced", size=1024) + os.mkdir("input/dir_replaced_with_file") + os.chmod("input/dir_replaced_with_file", stat.S_IFDIR | 0o755) + os.mkdir("input/dir_removed") + if are_symlinks_supported(): + os.mkdir("input/dir_replaced_with_link") + os.symlink("input/dir_replaced_with_file", "input/link_changed") + os.symlink("input/file_unchanged", "input/link_removed") + os.symlink("input/file_removed2", "input/link_target_removed") + os.symlink("input/empty", "input/link_target_contents_changed") + os.symlink("input/empty", "input/link_replaced_by_file") + if are_hardlinks_supported(): + os.link("input/file_replaced", "input/hardlink_target_replaced") + os.link("input/empty", "input/hardlink_contents_changed") + os.link("input/file_removed", "input/hardlink_removed") + os.link("input/file_removed2", "input/hardlink_target_removed") + + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + + # Create the first snapshot + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") + + # Setup files for the second snapshot + self.create_regular_file("file_added", size=2048) + self.create_regular_file("file_empty_added", size=0) + os.unlink("input/file_replaced") + self.create_regular_file("file_replaced", contents=b"0" * 4096) + os.unlink("input/file_removed") + os.unlink("input/file_removed2") + os.rmdir("input/dir_replaced_with_file") + self.create_regular_file("dir_replaced_with_file", size=8192) + os.chmod("input/dir_replaced_with_file", stat.S_IFREG | 0o755) + os.mkdir("input/dir_added") + os.rmdir("input/dir_removed") + if are_symlinks_supported(): + os.rmdir("input/dir_replaced_with_link") + os.symlink("input/dir_added", "input/dir_replaced_with_link") + os.unlink("input/link_changed") + os.symlink("input/dir_added", "input/link_changed") + os.symlink("input/dir_added", "input/link_added") + os.unlink("input/link_replaced_by_file") + self.create_regular_file("link_replaced_by_file", size=16384) + os.unlink("input/link_removed") + if are_hardlinks_supported(): + os.unlink("input/hardlink_removed") + os.link("input/file_added", "input/hardlink_added") + + with open("input/empty", "ab") as fd: + fd.write(b"appended_data") + + # Create the second snapshot + self.cmd(f"--repo={self.repository_location}", "create", "test1a", "input") + self.cmd(f"--repo={self.repository_location}", "create", "test1b", "input", "--chunker-params", "16,18,17,4095") + + def do_asserts(output, can_compare_ids): + # File contents changed (deleted and replaced with a new file) + change = "B" if can_compare_ids else "{:<19}".format("modified") + assert "file_replaced" in output # added to debug #3494 + assert f"{change} input/file_replaced" in output + + # File unchanged + assert "input/file_unchanged" not in output + + # Directory replaced with a regular file + if "BORG_TESTS_IGNORE_MODES" not in os.environ: + assert "[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file" in output + + # Basic directory cases + assert "added directory input/dir_added" in output + assert "removed directory input/dir_removed" in output + + if are_symlinks_supported(): + # Basic symlink cases + assert "changed link input/link_changed" in output + assert "added link input/link_added" in output + assert "removed link input/link_removed" in output + + # Symlink replacing or being replaced + assert "] input/dir_replaced_with_link" in output + assert "] input/link_replaced_by_file" in output + + # Symlink target removed. Should not affect the symlink at all. + assert "input/link_target_removed" not in output + + # The inode has two links and the file contents changed. Borg + # should notice the changes in both links. However, the symlink + # pointing to the file is not changed. + change = "0 B" if can_compare_ids else "{:<19}".format("modified") + assert f"{change} input/empty" in output + if are_hardlinks_supported(): + assert f"{change} input/hardlink_contents_changed" in output + if are_symlinks_supported(): + assert "input/link_target_contents_changed" not in output + + # Added a new file and a hard link to it. Both links to the same + # inode should appear as separate files. + assert "added 2.05 kB input/file_added" in output + if are_hardlinks_supported(): + assert "added 2.05 kB input/hardlink_added" in output + + # check if a diff between non-existent and empty new file is found + assert "added 0 B input/file_empty_added" in output + + # The inode has two links and both of them are deleted. They should + # appear as two deleted files. + assert "removed 256 B input/file_removed" in output + if are_hardlinks_supported(): + assert "removed 256 B input/hardlink_removed" in output + + # Another link (marked previously as the source in borg) to the + # same inode was removed. This should not change this link at all. + if are_hardlinks_supported(): + assert "input/hardlink_target_removed" not in output + + # Another link (marked previously as the source in borg) to the + # same inode was replaced with a new regular file. This should not + # change this link at all. + if are_hardlinks_supported(): + assert "input/hardlink_target_replaced" not in output + + def do_json_asserts(output, can_compare_ids): + def get_changes(filename, data): + chgsets = [j["changes"] for j in data if j["path"] == filename] + assert len(chgsets) < 2 + # return a flattened list of changes for given filename + return [chg for chgset in chgsets for chg in chgset] + + # convert output to list of dicts + joutput = [json.loads(line) for line in output.split("\n") if line] + + # File contents changed (deleted and replaced with a new file) + expected = {"type": "modified", "added": 4096, "removed": 1024} if can_compare_ids else {"type": "modified"} + assert expected in get_changes("input/file_replaced", joutput) + + # File unchanged + assert not any(get_changes("input/file_unchanged", joutput)) + + # Directory replaced with a regular file + if "BORG_TESTS_IGNORE_MODES" not in os.environ: + assert {"type": "mode", "old_mode": "drwxr-xr-x", "new_mode": "-rwxr-xr-x"} in get_changes( + "input/dir_replaced_with_file", joutput + ) + + # Basic directory cases + assert {"type": "added directory"} in get_changes("input/dir_added", joutput) + assert {"type": "removed directory"} in get_changes("input/dir_removed", joutput) + + if are_symlinks_supported(): + # Basic symlink cases + assert {"type": "changed link"} in get_changes("input/link_changed", joutput) + assert {"type": "added link"} in get_changes("input/link_added", joutput) + assert {"type": "removed link"} in get_changes("input/link_removed", joutput) + + # Symlink replacing or being replaced + assert any( + chg["type"] == "mode" and chg["new_mode"].startswith("l") + for chg in get_changes("input/dir_replaced_with_link", joutput) + ) + assert any( + chg["type"] == "mode" and chg["old_mode"].startswith("l") + for chg in get_changes("input/link_replaced_by_file", joutput) + ) + + # Symlink target removed. Should not affect the symlink at all. + assert not any(get_changes("input/link_target_removed", joutput)) + + # The inode has two links and the file contents changed. Borg + # should notice the changes in both links. However, the symlink + # pointing to the file is not changed. + expected = {"type": "modified", "added": 13, "removed": 0} if can_compare_ids else {"type": "modified"} + assert expected in get_changes("input/empty", joutput) + if are_hardlinks_supported(): + assert expected in get_changes("input/hardlink_contents_changed", joutput) + if are_symlinks_supported(): + assert not any(get_changes("input/link_target_contents_changed", joutput)) + + # Added a new file and a hard link to it. Both links to the same + # inode should appear as separate files. + assert {"type": "added", "size": 2048} in get_changes("input/file_added", joutput) + if are_hardlinks_supported(): + assert {"type": "added", "size": 2048} in get_changes("input/hardlink_added", joutput) + + # check if a diff between non-existent and empty new file is found + assert {"type": "added", "size": 0} in get_changes("input/file_empty_added", joutput) + + # The inode has two links and both of them are deleted. They should + # appear as two deleted files. + assert {"type": "removed", "size": 256} in get_changes("input/file_removed", joutput) + if are_hardlinks_supported(): + assert {"type": "removed", "size": 256} in get_changes("input/hardlink_removed", joutput) + + # Another link (marked previously as the source in borg) to the + # same inode was removed. This should not change this link at all. + if are_hardlinks_supported(): + assert not any(get_changes("input/hardlink_target_removed", joutput)) + + # Another link (marked previously as the source in borg) to the + # same inode was replaced with a new regular file. This should not + # change this link at all. + if are_hardlinks_supported(): + assert not any(get_changes("input/hardlink_target_replaced", joutput)) + + do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a"), True) + # We expect exit_code=1 due to the chunker params warning + do_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1b", exit_code=1), False) + do_json_asserts(self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1a", "--json-lines"), True) + + def test_sort_option(self): + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + + self.create_regular_file("a_file_removed", size=8) + self.create_regular_file("f_file_removed", size=16) + self.create_regular_file("c_file_changed", size=32) + self.create_regular_file("e_file_changed", size=64) + self.cmd(f"--repo={self.repository_location}", "create", "test0", "input") + + os.unlink("input/a_file_removed") + os.unlink("input/f_file_removed") + os.unlink("input/c_file_changed") + os.unlink("input/e_file_changed") + self.create_regular_file("c_file_changed", size=512) + self.create_regular_file("e_file_changed", size=1024) + self.create_regular_file("b_file_added", size=128) + self.create_regular_file("d_file_added", size=256) + self.cmd(f"--repo={self.repository_location}", "create", "test1", "input") + + output = self.cmd(f"--repo={self.repository_location}", "diff", "test0", "test1", "--sort") + expected = [ + "a_file_removed", + "b_file_added", + "c_file_changed", + "d_file_added", + "e_file_changed", + "f_file_removed", + ] + + assert all(x in line for x, line in zip(expected, output.splitlines()))