diff --git a/docs/development.rst b/docs/development.rst index 5db1f6d5a..7f38545d4 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -146,12 +146,9 @@ follow their `guide about avoiding ruining git blame`_: Continuous Integration ---------------------- -All pull requests go through `GitHub Actions`_, which runs the tests on Linux -and Mac OS X as well as the flake8 style checker. Windows builds run on AppVeyor_, -while additional Unix-like platforms are tested on Golem_. +All pull requests go through `GitHub Actions`_, which runs the tests on misc. +Python versions and on misc. platforms as well as some additional checks. -.. _AppVeyor: https://ci.appveyor.com/project/borgbackup/borg/ -.. _Golem: https://golem.enkore.de/view/Borg/ .. _GitHub Actions: https://github.com/borgbackup/borg/actions Output and Logging @@ -352,6 +349,8 @@ Checklist: scripts/upload-pypi X.Y.Z test scripts/upload-pypi X.Y.Z + Note: the signature is not uploaded to PyPi any more, but we upload it to + github releases. - Put binaries into dist/borg-OSNAME and sign them: :: @@ -370,9 +369,10 @@ Checklist: - Create a GitHub release, include: + * pypi dist package and signature * Standalone binaries (see above for how to create them). - + For OS X, document the OS X Fuse version in the README of the binaries. - OS X FUSE uses a kernel extension that needs to be compatible with the + + For macOS, document the macFUSE version in the README of the binaries. + macFUSE uses a kernel extension that needs to be compatible with the code contained in the binary. * A link to ``CHANGES.rst``. diff --git a/docs/installation.rst b/docs/installation.rst index fd6eeca1c..731c88a68 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -264,7 +264,7 @@ the installed ``openssl`` formula, point pkg-config to the correct path:: PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig" pip install borgbackup[llfuse] -For OS X Catalina and later, be aware that you must authorize full disk access. +Be aware that for all recent macOS releases you must authorize full disk access. It is no longer sufficient to run borg backups as root. If you have not yet granted full disk access, and you run Borg backup from cron, you will see messages such as:: diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index f0fc93231..761478763 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -384,8 +384,7 @@ The *tam* key is part of the :ref:`tertiary authentication mechanism 2 GB). -- Up to 1000 files per directory (10000 for repositories initialized with Borg 1.0) +- Up to 1000 files per directory. - rename(2) / MoveFile(Ex) should work as specified, i.e. on the same file system it should be a move (not a copy) operation, and in case of a directory it should fail if the destination exists and is not an empty directory, diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index 79d422543..520043c59 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -175,9 +175,7 @@ Separate compaction ~~~~~~~~~~~~~~~~~~~ Borg does not auto-compact the segment files in the repository at commit time -(at the end of each repository-writing command) any more. - -This is new since borg 1.2.0 and requires borg >= 1.2.0 on client and server. +(at the end of each repository-writing command) any more (since borg 1.2.0). This causes a similar behaviour of the repository as if it was in append-only mode (see below) most of the time (until ``borg compact`` is invoked or an @@ -236,7 +234,7 @@ in ``.ssh/authorized_keys``: command="borg serve --append-only ..." ssh-rsa command="borg serve ..." ssh-rsa -Running ``borg init`` via a ``borg serve --append-only`` server will *not* create +Running ``borg rcreate`` via a ``borg serve --append-only`` server will *not* create an append-only repository. Running ``borg rcreate --append-only`` creates an append-only repository regardless of server settings. @@ -276,7 +274,7 @@ with file 6:: That's all to do in the repository. -If you want to access this rollbacked repository from a client that already has +If you want to access this rolled back repository from a client that already has a cache for this repository, the cache will reflect a newer repository state than what you actually have in the repository now, after the rollback. diff --git a/setup.cfg b/setup.cfg index 327a38c7f..787627941 100644 --- a/setup.cfg +++ b/setup.cfg @@ -159,7 +159,7 @@ per_file_ignores = src/borg/testsuite/__init__.py:E501,F401 src/borg/testsuite/archive.py:E128,W504 src/borg/testsuite/archiver/__init__.py:E128,E501,E722,F401,F405,F811 - src/borg/testsuite/archiver/debug_cmds.py:E501 + src/borg/testsuite/archiver/debug_cmds.py:E501,F405 src/borg/testsuite/archiver/disk_full.py:F401,F405,F811 src/borg/testsuite/archiver/extract_cmd.py:F405 src/borg/testsuite/archiver/mount_cmds.py:E501,E722 diff --git a/setup_docs.py b/setup_docs.py index 41a41f35a..0861140e2 100644 --- a/setup_docs.py +++ b/setup_docs.py @@ -6,7 +6,7 @@ import re import sys import textwrap from collections import OrderedDict -from datetime import datetime +from datetime import datetime, timezone import time from setuptools import Command @@ -470,10 +470,8 @@ class build_man(Command): self.write_heading(write, description, double_sided=True) # man page metadata write(":Author: The Borg Collective") - write( - ":Date:", - datetime.utcfromtimestamp(int(os.environ.get("SOURCE_DATE_EPOCH", time.time()))).date().isoformat(), - ) + source_date_epoch = int(os.environ.get("SOURCE_DATE_EPOCH", time.time())) + write(":Date:", datetime.fromtimestamp(source_date_epoch, timezone.utc).date().isoformat()) write(":Manual section: 1") write(":Manual group: borg backup tool") write() diff --git a/src/borg/helpers/fs.py b/src/borg/helpers/fs.py index 68f0cd69f..14af58931 100644 --- a/src/borg/helpers/fs.py +++ b/src/borg/helpers/fs.py @@ -1,7 +1,7 @@ import errno import hashlib import os -import os.path +import posixpath import re import stat import subprocess @@ -231,7 +231,7 @@ def make_path_safe(path): path = path.lstrip("/") if path.startswith("../") or "/../" in path or path.endswith("/..") or path == "..": raise ValueError(f"unexpected '..' element in path {path!r}") - path = os.path.normpath(path) + path = posixpath.normpath(path) return path @@ -245,6 +245,11 @@ def remove_dotdot_prefixes(path): `path` is expected to be normalized already (e.g. via `os.path.normpath()`). """ + if is_win32: + if len(path) > 1 and path[1] == ":": + path = path.replace(":", "", 1) + path = path.replace("\\", "/") + path = path.lstrip("/") path = _dotdot_re.sub("", path) if path in ["", ".."]: diff --git a/src/borg/item.pyx b/src/borg/item.pyx index fac6ea493..5da91c2ed 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -562,7 +562,7 @@ cdef class ManifestItem(PropDict): archives = PropDictProperty(dict, 'dict of str -> dict') # name -> dict timestamp = PropDictProperty(str) config = PropDictProperty(dict) - item_keys = PropDictProperty(tuple, 'tuple of str') + item_keys = PropDictProperty(tuple, 'tuple of str') # legacy. new location is inside config. def update_internal(self, d): # legacy support for migration (data from old msgpacks comes in as bytes always, but sometimes we want str) @@ -650,7 +650,7 @@ class ItemDiff: self._can_compare_chunk_ids = can_compare_chunk_ids self._chunk_1 = chunk_1 self._chunk_2 = chunk_2 - + self._changes = {} if self._item1.is_link() or self._item2.is_link(): diff --git a/src/borg/manifest.py b/src/borg/manifest.py index c6409f4b7..189caf6a8 100644 --- a/src/borg/manifest.py +++ b/src/borg/manifest.py @@ -264,7 +264,9 @@ class Manifest: manifest.timestamp = m.get("timestamp") manifest.config = m.config # valid item keys are whatever is known in the repo or every key we know - manifest.item_keys = ITEM_KEYS | frozenset(m.get("item_keys", [])) + manifest.item_keys = ITEM_KEYS + manifest.item_keys |= frozenset(m.config.get("item_keys", [])) # new location of item_keys since borg2 + manifest.item_keys |= frozenset(m.get("item_keys", [])) # legacy: borg 1.x: item_keys not in config yet if manifest.tam_verified: manifest_required = manifest.config.get("tam_required", False) @@ -321,12 +323,12 @@ class Manifest: assert len(self.archives) <= MAX_ARCHIVES assert all(len(name) <= 255 for name in self.archives) assert len(self.item_keys) <= 100 + self.config["item_keys"] = tuple(sorted(self.item_keys)) manifest = ManifestItem( - version=1, + version=2, archives=StableDict(self.archives.get_raw_dict()), timestamp=self.timestamp, config=StableDict(self.config), - item_keys=tuple(sorted(self.item_keys)), ) self.tam_verified = True data = self.key.pack_and_authenticate_metadata(manifest.as_dict()) diff --git a/src/borg/testsuite/archiver/create_cmd.py b/src/borg/testsuite/archiver/create_cmd.py index b891604d6..4d02dbcb7 100644 --- a/src/borg/testsuite/archiver/create_cmd.py +++ b/src/borg/testsuite/archiver/create_cmd.py @@ -117,7 +117,6 @@ def test_basic_functionality(archivers, request): assert filter(info_output) == filter(info_output2) -@pytest.mark.skipif(is_win32, reason="still broken on windows") def test_archived_paths(archivers, request): archiver = request.getfixturevalue(archivers) repo_location = archiver.repository_location diff --git a/src/borg/testsuite/archiver/debug_cmds.py b/src/borg/testsuite/archiver/debug_cmds.py index aa9c264a4..161baf260 100644 --- a/src/borg/testsuite/archiver/debug_cmds.py +++ b/src/borg/testsuite/archiver/debug_cmds.py @@ -154,9 +154,10 @@ def test_debug_dump_manifest(archivers, request): result = json.load(f) assert "archives" in result assert "config" in result - assert "item_keys" in result assert "timestamp" in result assert "version" in result + assert "item_keys" in result["config"] + assert frozenset(result["config"]["item_keys"]) == ITEM_KEYS def test_debug_dump_archive(archivers, request): diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 0f96ad6cc..88936a9db 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -1202,6 +1202,11 @@ def test_swidth_slice_mixed_characters(): assert swidth_slice(string, 6) == "나윤a" +def utcfromtimestamp(timestamp): + """Returns a naive datetime instance representing the timestamp in the UTC timezone""" + return datetime.fromtimestamp(timestamp, timezone.utc).replace(tzinfo=None) + + def test_safe_timestamps(): if SUPPORT_32BIT_PLATFORMS: # ns fit into int64 @@ -1213,9 +1218,9 @@ def test_safe_timestamps(): # datetime won't fall over its y10k problem beyond_y10k = 2**100 with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) + utcfromtimestamp(beyond_y10k) + assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) + assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) else: # ns fit into int64 assert safe_ns(2**64) <= 2**63 - 1 @@ -1226,9 +1231,9 @@ def test_safe_timestamps(): # datetime won't fall over its y10k problem beyond_y10k = 2**100 with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) + utcfromtimestamp(beyond_y10k) + assert utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) + assert utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1) class TestPopenWithErrorHandling: diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 1f03c9a8f..598a8b5b6 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -161,7 +161,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_equal(acl_use_local_uid_gid(b"group:root:rw-:0"), b"group:0:rw-") -@unittest.skipUnless(sys.platform.startswith("darwin"), "OS X only test") +@unittest.skipUnless(sys.platform.startswith("darwin"), "macOS only test") @unittest.skipIf(fakeroot_detected(), "not compatible with fakeroot") class PlatformDarwinTestCase(BaseTestCase): def setUp(self): diff --git a/src/borg/xattr.py b/src/borg/xattr.py index f3bd27610..b914c5046 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -1,4 +1,4 @@ -"""A basic extended attributes (xattr) implementation for Linux, FreeBSD and MacOS X.""" +"""A basic extended attributes (xattr) implementation for Linux, FreeBSD and macOS.""" import errno import os