From 15781efab349cc87ce49b508e186cada7ed337e6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 20:11:26 +0200 Subject: [PATCH 01/12] docs: CI only on github actions now --- docs/development.rst | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 5db1f6d5a..8eb9d929b 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -146,12 +146,9 @@ follow their `guide about avoiding ruining git blame`_: Continuous Integration ---------------------- -All pull requests go through `GitHub Actions`_, which runs the tests on Linux -and Mac OS X as well as the flake8 style checker. Windows builds run on AppVeyor_, -while additional Unix-like platforms are tested on Golem_. +All pull requests go through `GitHub Actions`_, which runs the tests on misc. +Python versions and on misc. platforms as well as some additional checks. -.. _AppVeyor: https://ci.appveyor.com/project/borgbackup/borg/ -.. _Golem: https://golem.enkore.de/view/Borg/ .. _GitHub Actions: https://github.com/borgbackup/borg/actions Output and Logging From 5fb4cc3b4b4cdef9220f413ea94b510f88918f4d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 20:20:35 +0200 Subject: [PATCH 02/12] docs: no signature upload on pypi any more but we upload it on github releases, so people can check it. --- docs/development.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/development.rst b/docs/development.rst index 8eb9d929b..0bbf03923 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -349,6 +349,8 @@ Checklist: scripts/upload-pypi X.Y.Z test scripts/upload-pypi X.Y.Z + Note: the signature is not uploaded to PyPi any more, but we upload it to + github releases. - Put binaries into dist/borg-OSNAME and sign them: :: @@ -367,6 +369,7 @@ Checklist: - Create a GitHub release, include: + * pypi dist package and signature * Standalone binaries (see above for how to create them). + For OS X, document the OS X Fuse version in the README of the binaries. From 54b654e3b12091e34077fb6501f5e43472e4c216 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Sun, 25 Jun 2023 17:38:54 +0100 Subject: [PATCH 03/12] fs: Properly normalise paths on Windows Use forward slashes and integrate the drive letter into the path. --- src/borg/helpers/fs.py | 9 +++++++-- src/borg/testsuite/archiver/create_cmd.py | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 68f0cd69f..14af58931 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -1,7 +1,7 @@ import errno import hashlib import os -import os.path +import posixpath import re import stat import subprocess @@ -231,7 +231,7 @@ def make_path_safe(path): path = path.lstrip("/") if path.startswith("../") or "/../" in path or path.endswith("/..") or path == "..": raise ValueError(f"unexpected '..' element in path {path!r}") - path = os.path.normpath(path) + path = posixpath.normpath(path) return path @@ -245,6 +245,11 @@ def remove_dotdot_prefixes(path): `path` is expected to be normalized already (e.g. via `os.path.normpath()`). """ + if is_win32: + if len(path) > 1 and path[1] == ":": + path = path.replace(":", "", 1) + path = path.replace("\\", "/") + path = path.lstrip("/") path = _dotdot_re.sub("", path) if path in ["", ".."]: diff --git a/src/borg/testsuite/archiver/create_cmd.py b/src/borg/testsuite/archiver/create_cmd.py index cd80ea486..667c2d9ee 100644 --- a/src/borg/testsuite/archiver/create_cmd.py +++ b/src/borg/testsuite/archiver/create_cmd.py @@ -110,8 +110,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): # To make our lives easier and to support cross-platform extraction we always use slashes. # Similarly, archived paths are expected to be full, but relative (have no leading slash). full_path = os.path.abspath(os.path.join(self.input_path, "test")) - # remove windows drive letter, if any: - posix_path = full_path[2:] if full_path[1] == ":" else full_path + # remove colon from windows drive letter, if any: + posix_path = full_path.replace(":", "") if full_path[1] == ":" else full_path # only needed on windows in case there are backslashes: posix_path = posix_path.replace("\\", "/") # no leading slash in borg archives: From 7e0cdd7c5e7541393d93b798694b665c6dc43001 Mon Sep 17 00:00:00 2001 From: Rayyan Ansari Date: Sun, 25 Jun 2023 17:40:17 +0100 Subject: [PATCH 04/12] testsuite: Enable test_archived_paths on Windows --- src/borg/testsuite/archiver/create_cmd.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/borg/testsuite/archiver/create_cmd.py b/src/borg/testsuite/archiver/create_cmd.py index 667c2d9ee..bf9f9c71d 100644 --- a/src/borg/testsuite/archiver/create_cmd.py +++ b/src/borg/testsuite/archiver/create_cmd.py @@ -103,7 +103,6 @@ class ArchiverTestCase(ArchiverTestCaseBase): # the interesting parts of info_output2 and info_output should be same self.assert_equal(filter(info_output), filter(info_output2)) - @pytest.mark.skipif(is_win32, reason="still broken on windows") def test_archived_paths(self): # As borg comes from the POSIX (Linux, UNIX) world, a lot of stuff assumes path separators # to be slashes "/", e.g.: in archived items, for pattern matching. From 84b1ad6948833ae8621d306deeb0db55010ce8fa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 23:32:36 +0200 Subject: [PATCH 05/12] docs: remove info about borg 1.0 files per dir --- docs/usage/general/file-systems.rst.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage/general/file-systems.rst.inc b/docs/usage/general/file-systems.rst.inc index 7a022899c..1fbe47246 100644 --- a/docs/usage/general/file-systems.rst.inc +++ b/docs/usage/general/file-systems.rst.inc @@ -21,7 +21,7 @@ and readable after one of the failures mentioned above occurred, run - At least three directory levels with short names - Typically, file sizes up to a few hundred MB. Large repositories may require large files (>2 GB). -- Up to 1000 files per directory (10000 for repositories initialized with Borg 1.0) +- Up to 1000 files per directory. - rename(2) / MoveFile(Ex) should work as specified, i.e. on the same file system it should be a move (not a copy) operation, and in case of a directory it should fail if the destination exists and is not an empty directory, From 89cfd083ba58e64aa14d4ed6030dda1f1d30e537 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 23:36:41 +0200 Subject: [PATCH 06/12] docs: minor fixes/update to notes --- docs/usage/notes.rst | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index 79d422543..520043c59 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -175,9 +175,7 @@ Separate compaction ~~~~~~~~~~~~~~~~~~~ Borg does not auto-compact the segment files in the repository at commit time -(at the end of each repository-writing command) any more. - -This is new since borg 1.2.0 and requires borg >= 1.2.0 on client and server. +(at the end of each repository-writing command) any more (since borg 1.2.0). This causes a similar behaviour of the repository as if it was in append-only mode (see below) most of the time (until ``borg compact`` is invoked or an @@ -236,7 +234,7 @@ in ``.ssh/authorized_keys``: command="borg serve --append-only ..." ssh-rsa command="borg serve ..." ssh-rsa -Running ``borg init`` via a ``borg serve --append-only`` server will *not* create +Running ``borg rcreate`` via a ``borg serve --append-only`` server will *not* create an append-only repository. Running ``borg rcreate --append-only`` creates an append-only repository regardless of server settings. @@ -276,7 +274,7 @@ with file 6:: That's all to do in the repository. -If you want to access this rollbacked repository from a client that already has +If you want to access this rolled back repository from a client that already has a cache for this repository, the cache will reflect a newer repository state than what you actually have in the repository now, after the rollback. From ddd93c5828e52c40643a325119ff552326a1a7eb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 23:39:44 +0200 Subject: [PATCH 07/12] docs: remove --chunker-params hint to borg < 1.0 --- docs/internals/data-structures.rst | 2 +- docs/usage/create.rst | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index f0fc93231..c8d819599 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -793,7 +793,7 @@ For small hash tables, we start with a growth factor of 2, which comes down to E.g. backing up a total count of 1 Mi (IEC binary prefix i.e. 2^20) files with a total size of 1TiB. -a) with ``create --chunker-params buzhash,10,23,16,4095`` (custom, like borg < 1.0): +a) with ``create --chunker-params buzhash,10,23,16,4095`` (custom): mem_usage = 2.8GiB diff --git a/docs/usage/create.rst b/docs/usage/create.rst index 8bc4c89a4..f2ff4966a 100644 --- a/docs/usage/create.rst +++ b/docs/usage/create.rst @@ -36,8 +36,7 @@ Examples $ fusermount -u sshfs-mount # Make a big effort in fine granular deduplication (big chunk management - # overhead, needs a lot of RAM and disk space, see formula in internals - # docs - same parameters as borg < 1.0): + # overhead, needs a lot of RAM and disk space, see formula in internals docs): $ borg create --chunker-params buzhash,10,23,16,4095 small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) From 96c3c902308e30b091d6b0705221feb4f0783853 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 23:46:53 +0200 Subject: [PATCH 08/12] docs: remove hint about item_keys in manifest --- docs/internals/data-structures.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index c8d819599..761478763 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -384,8 +384,7 @@ The *tam* key is part of the :ref:`tertiary authentication mechanism Date: Tue, 4 Jul 2023 23:55:06 +0200 Subject: [PATCH 09/12] docs: update macOS hint about full disk access --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index fd6eeca1c..731c88a68 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -264,7 +264,7 @@ the installed ``openssl`` formula, point pkg-config to the correct path:: PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" pip install borgbackup[llfuse] -For OS X Catalina and later, be aware that you must authorize full disk access. +Be aware that for all recent macOS releases you must authorize full disk access. It is no longer sufficient to run borg backups as root. If you have not yet granted full disk access, and you run Borg backup from cron, you will see messages such as:: From 8db8fd601b82df885db0becfffb87c7f2d522c78 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 4 Jul 2023 23:56:29 +0200 Subject: [PATCH 10/12] docs: OS X -> macOS --- docs/development.rst | 4 ++-- docs/usage/general/file-metadata.rst.inc | 2 +- src/borg/testsuite/platform.py | 2 +- src/borg/xattr.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/development.rst b/docs/development.rst index 0bbf03923..7f38545d4 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -372,7 +372,7 @@ Checklist: * pypi dist package and signature * Standalone binaries (see above for how to create them). - + For OS X, document the OS X Fuse version in the README of the binaries. - OS X FUSE uses a kernel extension that needs to be compatible with the + + For macOS, document the macFUSE version in the README of the binaries. + macFUSE uses a kernel extension that needs to be compatible with the code contained in the binary. * A link to ``CHANGES.rst``. diff --git a/docs/usage/general/file-metadata.rst.inc b/docs/usage/general/file-metadata.rst.inc index c2694d188..c2921819e 100644 --- a/docs/usage/general/file-metadata.rst.inc +++ b/docs/usage/general/file-metadata.rst.inc @@ -29,7 +29,7 @@ On some platforms additional features are supported: +=========================+==========+===========+============+ | Linux | Yes | Yes | Yes [1]_ | +-------------------------+----------+-----------+------------+ -| Mac OS X | Yes | Yes | Yes (all) | +| macOS | Yes | Yes | Yes (all) | +-------------------------+----------+-----------+------------+ | FreeBSD | Yes | Yes | Yes (all) | +-------------------------+----------+-----------+------------+ diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 1f03c9a8f..598a8b5b6 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -161,7 +161,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_equal(acl_use_local_uid_gid(b"group:root:rw-:0"), b"group:0:rw-") -@unittest.skipUnless(sys.platform.startswith("darwin"), "OS X only test") +@unittest.skipUnless(sys.platform.startswith("darwin"), "macOS only test") @unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot") class PlatformDarwinTestCase(BaseTestCase): def setUp(self): diff --git a/src/borg/xattr.py b/src/borg/xattr.py index f3bd27610..b914c5046 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -1,4 +1,4 @@ -"""A basic extended attributes (xattr) implementation for Linux, FreeBSD and MacOS X.""" +"""A basic extended attributes (xattr) implementation for Linux, FreeBSD and macOS.""" import errno import os From 51e68c24e43ce1bb5fbd00645b4e5f590214ccd2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 5 Jul 2023 01:11:24 +0200 Subject: [PATCH 11/12] manifest: move item_keys into config dict, fixes #7710 also: manifest.version == 2 now --- setup.cfg | 2 +- src/borg/item.pyx | 4 ++-- src/borg/manifest.py | 8 +++++--- src/borg/testsuite/archiver/debug_cmds.py | 3 ++- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 327a38c7f..787627941 100644 --- a/setup.cfg +++ b/setup.cfg @@ -159,7 +159,7 @@ per_file_ignores = src/borg/testsuite/__init__.py:E501,F401 src/borg/testsuite/archive.py:E128,W504 src/borg/testsuite/archiver/__init__.py:E128,E501,E722,F401,F405,F811 - src/borg/testsuite/archiver/debug_cmds.py:E501 + src/borg/testsuite/archiver/debug_cmds.py:E501,F405 src/borg/testsuite/archiver/disk_full.py:F401,F405,F811 src/borg/testsuite/archiver/extract_cmd.py:F405 src/borg/testsuite/archiver/mount_cmds.py:E501,E722 diff --git a/src/borg/item.pyx b/src/borg/item.pyx index fac6ea493..5da91c2ed 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -562,7 +562,7 @@ cdef class ManifestItem(PropDict): archives = PropDictProperty(dict, 'dict of str -> dict') # name -> dict timestamp = PropDictProperty(str) config = PropDictProperty(dict) - item_keys = PropDictProperty(tuple, 'tuple of str') + item_keys = PropDictProperty(tuple, 'tuple of str') # legacy. new location is inside config. def update_internal(self, d): # legacy support for migration (data from old msgpacks comes in as bytes always, but sometimes we want str) @@ -650,7 +650,7 @@ class ItemDiff: self._can_compare_chunk_ids = can_compare_chunk_ids self._chunk_1 = chunk_1 self._chunk_2 = chunk_2 - + self._changes = {} if self._item1.is_link() or self._item2.is_link(): diff --git a/src/borg/manifest.py b/src/borg/manifest.py index c6409f4b7..189caf6a8 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -264,7 +264,9 @@ class Manifest: manifest.timestamp = m.get("timestamp") manifest.config = m.config # valid item keys are whatever is known in the repo or every key we know - manifest.item_keys = ITEM_KEYS | frozenset(m.get("item_keys", [])) + manifest.item_keys = ITEM_KEYS + manifest.item_keys |= frozenset(m.config.get("item_keys", [])) # new location of item_keys since borg2 + manifest.item_keys |= frozenset(m.get("item_keys", [])) # legacy: borg 1.x: item_keys not in config yet if manifest.tam_verified: manifest_required = manifest.config.get("tam_required", False) @@ -321,12 +323,12 @@ class Manifest: assert len(self.archives) <= MAX_ARCHIVES assert all(len(name) <= 255 for name in self.archives) assert len(self.item_keys) <= 100 + self.config["item_keys"] = tuple(sorted(self.item_keys)) manifest = ManifestItem( - version=1, + version=2, archives=StableDict(self.archives.get_raw_dict()), timestamp=self.timestamp, config=StableDict(self.config), - item_keys=tuple(sorted(self.item_keys)), ) self.tam_verified = True data = self.key.pack_and_authenticate_metadata(manifest.as_dict()) diff --git a/src/borg/testsuite/archiver/debug_cmds.py b/src/borg/testsuite/archiver/debug_cmds.py index ee22da558..6f7c251e5 100644 --- a/src/borg/testsuite/archiver/debug_cmds.py +++ b/src/borg/testsuite/archiver/debug_cmds.py @@ -133,7 +133,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): result = json.load(f) assert "archives" in result assert "config" in result - assert "item_keys" in result + assert "item_keys" in result["config"] + assert frozenset(result["config"]["item_keys"]) == ITEM_KEYS assert "timestamp" in result assert "version" in result From ae0b3d2fffd2dfd18c7726c15afded174ded75d4 Mon Sep 17 00:00:00 2001 From: Felix Schwarz Date: Thu, 6 Jul 2023 21:26:40 +0200 Subject: [PATCH 12/12] replace "datetime.utcfromtimestamp" with custom helper to avoid deprecation warnings when using Python 3.12 --- setup_docs.py | 8 +++----- src/borg/testsuite/helpers.py | 17 +++++++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/setup_docs.py b/setup_docs.py index 41a41f35a..0861140e2 100644 --- a/setup_docs.py +++ b/setup_docs.py @@ -6,7 +6,7 @@ import re import sys import textwrap from collections import OrderedDict -from datetime import datetime +from datetime import datetime, timezone import time from setuptools import Command @@ -470,10 +470,8 @@ class build_man(Command): self.write_heading(write, description, double_sided=True) # man page metadata write(":Author: The Borg Collective") - write( - ":Date:", - datetime.utcfromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH", time.time()))).date().isoformat(), - ) + source_date_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) + write(":Date:", datetime.fromtimestamp(source_date_epoch, timezone.utc).date().isoformat()) write(":Manual section: 1") write(":Manual group: borg backup tool") write() diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 0f96ad6cc..88936a9db 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -1202,6 +1202,11 @@ def test_swidth_slice_mixed_characters(): assert swidth_slice(string, 6) == "나윤a" +def utcfromtimestamp(timestamp): + """Returns a naive datetime instance representing the timestamp in the UTC timezone""" + return datetime.fromtimestamp(timestamp, timezone.utc).replace(tzinfo=None) + + def test_safe_timestamps(): if SUPPORT_32BIT_PLATFORMS: # ns fit into int64 @@ -1213,9 +1218,9 @@ def test_safe_timestamps(): # datetime won't fall over its y10k problem beyond_y10k = 2**100 with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) + utcfromtimestamp(beyond_y10k) + assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) + assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) else: # ns fit into int64 assert safe_ns(2**64) <= 2**63 - 1 @@ -1226,9 +1231,9 @@ def test_safe_timestamps(): # datetime won't fall over its y10k problem beyond_y10k = 2**100 with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) + utcfromtimestamp(beyond_y10k) + assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) + assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) class TestPopenWithErrorHandling: