diff --git a/src/borg/helpers.py b/src/borg/helpers.py index a52ff26bf..8bc9959e9 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -905,10 +905,30 @@ def SortBySpec(text): # Not too rarely, we get crappy timestamps from the fs, that overflow some computations. -# As they are crap anyway, nothing is lost if we just clamp them to the max valid value. -# msgpack can only pack uint64. datetime is limited to year 9999. -MAX_NS = 18446744073000000000 # less than 2**64 - 1 ns. also less than y9999. -MAX_S = MAX_NS // 1000000000 +# As they are crap anyway (valid filesystem timestamps always refer to the past up to +# the present, but never to the future), nothing is lost if we just clamp them to the +# maximum value we can support. +# As long as people are using borg on 32bit platforms to access borg archives, we must +# keep this value True. But we can expect that we can stop supporting 32bit platforms +# well before coming close to the year 2038, so this will never be a practical problem. +SUPPORT_32BIT_PLATFORMS = True # set this to False before y2038. + +if SUPPORT_32BIT_PLATFORMS: + # second timestamps will fit into a signed int32 (platform time_t limit). + # nanosecond timestamps thus will naturally fit into a signed int64. + # subtract last 48h to avoid any issues that could be caused by tz calculations. + # this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit). + # msgpack can pack up to uint64. + MAX_S = 2**31-1 - 48*3600 + MAX_NS = MAX_S * 1000000000 +else: + # nanosecond timestamps will fit into a signed int64. + # subtract last 48h to avoid any issues that could be caused by tz calculations. + # this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit). + # round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S. + # msgpack can pack up to uint64. + MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000 + MAX_S = MAX_NS // 1000000000 def safe_s(ts): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 366900d3e..c065feb16 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -38,6 +38,7 @@ from ..helpers import Manifest from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..helpers import bin_to_hex from ..helpers import IECommand +from ..helpers import MAX_S from ..item import Item from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError from ..keymanager import RepoIdMismatch, NotABorgKeyFile @@ -293,12 +294,7 @@ class ArchiverTestCaseBase(BaseTestCase): """ # File self.create_regular_file('empty', size=0) - # next code line raises OverflowError on 32bit cpu (raspberry pi 2): - # 2600-01-01 > 2**64 ns - # os.utime('input/empty', (19880895600, 19880895600)) - # thus, we better test with something not that far in future: - # 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline": - os.utime('input/empty', (2**31 - 1, 2**31 - 1)) + os.utime('input/empty', (MAX_S, MAX_S)) self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('flagfile', size=1024) # Directory diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index f5db0992d..047e41c84 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -28,7 +28,7 @@ from ..helpers import parse_pattern, PatternMatcher from ..helpers import PathFullPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, RegexPattern from ..helpers import swidth_slice from ..helpers import chunkit -from ..helpers import safe_ns, safe_s +from ..helpers import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS from . import BaseTestCase, FakeInputs @@ -1250,15 +1250,29 @@ def test_swidth_slice_mixed_characters(): def test_safe_timestamps(): - # ns fit into uint64 - assert safe_ns(2 ** 64) < 2 ** 64 - assert safe_ns(-1) == 0 - # s are so that their ns conversion fits into uint64 - assert safe_s(2 ** 64) * 1000000000 < 2 ** 64 - assert safe_s(-1) == 0 - # datetime won't fall over its y10k problem - beyond_y10k = 2 ** 100 - with pytest.raises(OverflowError): - datetime.utcfromtimestamp(beyond_y10k) - assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31) - assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31) + if SUPPORT_32BIT_PLATFORMS: + # ns fit into int64 + assert safe_ns(2 ** 64) <= 2 ** 63 - 1 + assert safe_ns(-1) == 0 + # s fit into int32 + assert safe_s(2 ** 64) <= 2 ** 31 - 1 + assert safe_s(-1) == 0 + # datetime won't fall over its y10k problem + beyond_y10k = 2 ** 100 + with pytest.raises(OverflowError): + datetime.utcfromtimestamp(beyond_y10k) + assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1) + assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1) + else: + # ns fit into int64 + assert safe_ns(2 ** 64) <= 2 ** 63 - 1 + assert safe_ns(-1) == 0 + # s are so that their ns conversion fits into int64 + assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1 + assert safe_s(-1) == 0 + # datetime won't fall over its y10k problem + beyond_y10k = 2 ** 100 + with pytest.raises(OverflowError): + datetime.utcfromtimestamp(beyond_y10k) + assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1) + assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1)