Merge pull request #2404 from ThomasWaldmann/limit-y2038

embrace y2038 issue to support 32bit platforms

src/borg/helpers.py

@@ -905,10 +905,30 @@ def SortBySpec(text):
 
 
 # Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
-# As they are crap anyway, nothing is lost if we just clamp them to the max valid value.
-# msgpack can only pack uint64. datetime is limited to year 9999.
-MAX_NS = 18446744073000000000  # less than 2**64 - 1 ns. also less than y9999.
-MAX_S = MAX_NS // 1000000000
+# As they are crap anyway (valid filesystem timestamps always refer to the past up to
+# the present, but never to the future), nothing is lost if we just clamp them to the
+# maximum value we can support.
+# As long as people are using borg on 32bit platforms to access borg archives, we must
+# keep this value True. But we can expect that we can stop supporting 32bit platforms
+# well before coming close to the year 2038, so this will never be a practical problem.
+SUPPORT_32BIT_PLATFORMS = True  # set this to False before y2038.
+
+if SUPPORT_32BIT_PLATFORMS:
+    # second timestamps will fit into a signed int32 (platform time_t limit).
+    # nanosecond timestamps thus will naturally fit into a signed int64.
+    # subtract last 48h to avoid any issues that could be caused by tz calculations.
+    # this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit).
+    # msgpack can pack up to uint64.
+    MAX_S = 2**31-1 - 48*3600
+    MAX_NS = MAX_S * 1000000000
+else:
+    # nanosecond timestamps will fit into a signed int64.
+    # subtract last 48h to avoid any issues that could be caused by tz calculations.
+    # this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit).
+    # round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S.
+    # msgpack can pack up to uint64.
+    MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000
+    MAX_S = MAX_NS // 1000000000
 
 
 def safe_s(ts):

src/borg/testsuite/archiver.py

@@ -38,6 +38,7 @@
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
 from ..helpers import bin_to_hex
 from ..helpers import IECommand
+from ..helpers import MAX_S
 from ..item import Item
 from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
 from ..keymanager import RepoIdMismatch, NotABorgKeyFile
@@ -293,12 +294,7 @@ def create_test_files(self):
         """
         # File
         self.create_regular_file('empty', size=0)
-        # next code line raises OverflowError on 32bit cpu (raspberry pi 2):
-        # 2600-01-01 > 2**64 ns
-        # os.utime('input/empty', (19880895600, 19880895600))
-        # thus, we better test with something not that far in future:
-        # 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
-        os.utime('input/empty', (2**31 - 1, 2**31 - 1))
+        os.utime('input/empty', (MAX_S, MAX_S))
         self.create_regular_file('file1', size=1024 * 80)
         self.create_regular_file('flagfile', size=1024)
         # Directory

src/borg/testsuite/helpers.py

@@ -28,7 +28,7 @@
 from ..helpers import PathFullPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, RegexPattern
 from ..helpers import swidth_slice
 from ..helpers import chunkit
-from ..helpers import safe_ns, safe_s
+from ..helpers import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS
 
 from . import BaseTestCase, FakeInputs
 
@@ -1250,15 +1250,29 @@ def test_swidth_slice_mixed_characters():
 
 
 def test_safe_timestamps():
-    # ns fit into uint64
-    assert safe_ns(2 ** 64) < 2 ** 64
-    assert safe_ns(-1) == 0
-    # s are so that their ns conversion fits into uint64
-    assert safe_s(2 ** 64) * 1000000000 < 2 ** 64
-    assert safe_s(-1) == 0
-    # datetime won't fall over its y10k problem
-    beyond_y10k = 2 ** 100
-    with pytest.raises(OverflowError):
-        datetime.utcfromtimestamp(beyond_y10k)
-    assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31)
-    assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31)
+    if SUPPORT_32BIT_PLATFORMS:
+        # ns fit into int64
+        assert safe_ns(2 ** 64) <= 2 ** 63 - 1
+        assert safe_ns(-1) == 0
+        # s fit into int32
+        assert safe_s(2 ** 64) <= 2 ** 31 - 1
+        assert safe_s(-1) == 0
+        # datetime won't fall over its y10k problem
+        beyond_y10k = 2 ** 100
+        with pytest.raises(OverflowError):
+            datetime.utcfromtimestamp(beyond_y10k)
+        assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1)
+        assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1)
+    else:
+        # ns fit into int64
+        assert safe_ns(2 ** 64) <= 2 ** 63 - 1
+        assert safe_ns(-1) == 0
+        # s are so that their ns conversion fits into int64
+        assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1
+        assert safe_s(-1) == 0
+        # datetime won't fall over its y10k problem
+        beyond_y10k = 2 ** 100
+        with pytest.raises(OverflowError):
+            datetime.utcfromtimestamp(beyond_y10k)
+        assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1)
+        assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1)