1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-25 17:27:31 +00:00

embrace y2038 issue to support 32bit platforms

This commit is contained in:
Thomas Waldmann 2017-04-08 15:46:24 +02:00
parent 6f47b797f9
commit de76a6b821
3 changed files with 53 additions and 23 deletions

View file

@ -905,10 +905,30 @@ def SortBySpec(text):
# Not too rarely, we get crappy timestamps from the fs, that overflow some computations. # Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
# As they are crap anyway, nothing is lost if we just clamp them to the max valid value. # As they are crap anyway (valid filesystem timestamps always refer to the past up to
# msgpack can only pack uint64. datetime is limited to year 9999. # the present, but never to the future), nothing is lost if we just clamp them to the
MAX_NS = 18446744073000000000 # less than 2**64 - 1 ns. also less than y9999. # maximum value we can support.
MAX_S = MAX_NS // 1000000000 # As long as people are using borg on 32bit platforms to access borg archives, we must
# keep this value True. But we can expect that we can stop supporting 32bit platforms
# well before coming close to the year 2038, so this will never be a practical problem.
SUPPORT_32BIT_PLATFORMS = True # set this to False before y2038.
if SUPPORT_32BIT_PLATFORMS:
# second timestamps will fit into a signed int32 (platform time_t limit).
# nanosecond timestamps thus will naturally fit into a signed int64.
# subtract last 48h to avoid any issues that could be caused by tz calculations.
# this is in the year 2038, so it is also less than y9999 (which is a datetime internal limit).
# msgpack can pack up to uint64.
MAX_S = 2**31-1 - 48*3600
MAX_NS = MAX_S * 1000000000
else:
# nanosecond timestamps will fit into a signed int64.
# subtract last 48h to avoid any issues that could be caused by tz calculations.
# this is in the year 2262, so it is also less than y9999 (which is a datetime internal limit).
# round down to 1e9 multiple, so MAX_NS corresponds precisely to a integer MAX_S.
# msgpack can pack up to uint64.
MAX_NS = (2**63-1 - 48*3600*1000000000) // 1000000000 * 1000000000
MAX_S = MAX_NS // 1000000000
def safe_s(ts): def safe_s(ts):

View file

@ -38,6 +38,7 @@
from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
from ..helpers import bin_to_hex from ..helpers import bin_to_hex
from ..helpers import IECommand from ..helpers import IECommand
from ..helpers import MAX_S
from ..item import Item from ..item import Item
from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError from ..key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..keymanager import RepoIdMismatch, NotABorgKeyFile
@ -293,12 +294,7 @@ def create_test_files(self):
""" """
# File # File
self.create_regular_file('empty', size=0) self.create_regular_file('empty', size=0)
# next code line raises OverflowError on 32bit cpu (raspberry pi 2): os.utime('input/empty', (MAX_S, MAX_S))
# 2600-01-01 > 2**64 ns
# os.utime('input/empty', (19880895600, 19880895600))
# thus, we better test with something not that far in future:
# 2038-01-19 (1970 + 2^31 - 1 seconds) is the 32bit "deadline":
os.utime('input/empty', (2**31 - 1, 2**31 - 1))
self.create_regular_file('file1', size=1024 * 80) self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('flagfile', size=1024) self.create_regular_file('flagfile', size=1024)
# Directory # Directory

View file

@ -28,7 +28,7 @@
from ..helpers import PathFullPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, RegexPattern from ..helpers import PathFullPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, RegexPattern
from ..helpers import swidth_slice from ..helpers import swidth_slice
from ..helpers import chunkit from ..helpers import chunkit
from ..helpers import safe_ns, safe_s from ..helpers import safe_ns, safe_s, SUPPORT_32BIT_PLATFORMS
from . import BaseTestCase, FakeInputs from . import BaseTestCase, FakeInputs
@ -1250,15 +1250,29 @@ def test_swidth_slice_mixed_characters():
def test_safe_timestamps(): def test_safe_timestamps():
# ns fit into uint64 if SUPPORT_32BIT_PLATFORMS:
assert safe_ns(2 ** 64) < 2 ** 64 # ns fit into int64
assert safe_ns(-1) == 0 assert safe_ns(2 ** 64) <= 2 ** 63 - 1
# s are so that their ns conversion fits into uint64 assert safe_ns(-1) == 0
assert safe_s(2 ** 64) * 1000000000 < 2 ** 64 # s fit into int32
assert safe_s(-1) == 0 assert safe_s(2 ** 64) <= 2 ** 31 - 1
# datetime won't fall over its y10k problem assert safe_s(-1) == 0
beyond_y10k = 2 ** 100 # datetime won't fall over its y10k problem
with pytest.raises(OverflowError): beyond_y10k = 2 ** 100
datetime.utcfromtimestamp(beyond_y10k) with pytest.raises(OverflowError):
assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31) datetime.utcfromtimestamp(beyond_y10k)
assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31) assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2038, 1, 1)
assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2038, 1, 1)
else:
# ns fit into int64
assert safe_ns(2 ** 64) <= 2 ** 63 - 1
assert safe_ns(-1) == 0
# s are so that their ns conversion fits into int64
assert safe_s(2 ** 64) * 1000000000 <= 2 ** 63 - 1
assert safe_s(-1) == 0
# datetime won't fall over its y10k problem
beyond_y10k = 2 ** 100
with pytest.raises(OverflowError):
datetime.utcfromtimestamp(beyond_y10k)
assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2262, 1, 1)
assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2262, 1, 1)