From 07bcd29144d9c3b3bf557277a68624b0b412e705 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 15 Mar 2017 18:54:34 +0100 Subject: [PATCH] clamp (nano)second values to unproblematic range, fixes #2304 filesystem -> clamp -> archive (create) (cherry picked from commit b7a17a6db76283c9d9d6f246831c3010fbb35fe2) --- borg/archive.py | 8 ++++---- borg/cache.py | 4 ++-- borg/helpers.py | 34 ++++++++++++++++++++++++++++------ borg/testsuite/helpers.py | 18 +++++++++++++++++- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 8c1eabcf7..91a15019d 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -19,7 +19,7 @@ from . import xattr from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \ parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \ Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ - ProgressIndicatorPercent, IntegrityError, set_ec, EXIT_WARNING + ProgressIndicatorPercent, IntegrityError, set_ec, EXIT_WARNING, safe_ns from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex @@ -588,15 +588,15 @@ Number of files: {0.stats.nfiles}'''.format( b'mode': st.st_mode, b'uid': st.st_uid, b'user': uid2user(st.st_uid), b'gid': st.st_gid, b'group': gid2group(st.st_gid), - b'mtime': int_to_bigint(st.st_mtime_ns), + b'mtime': int_to_bigint(safe_ns(st.st_mtime_ns)), } # borg can work with archives only having mtime (older attic archives do not have # atime/ctime). it can be useful to omit atime/ctime, if they change without the # file content changing - e.g. to get better metadata deduplication. if not self.noatime: - item[b'atime'] = int_to_bigint(st.st_atime_ns) + item[b'atime'] = int_to_bigint(safe_ns(st.st_atime_ns)) if not self.noctime: - item[b'ctime'] = int_to_bigint(st.st_ctime_ns) + item[b'ctime'] = int_to_bigint(safe_ns(st.st_ctime_ns)) if self.numeric_owner: item[b'user'] = item[b'group'] = None with backup_io(): diff --git a/borg/cache.py b/borg/cache.py index 948f67d86..ef87984dc 100644 --- a/borg/cache.py +++ b/borg/cache.py @@ -10,7 +10,7 @@ from .key import PlaintextKey from .logger import create_logger logger = create_logger() from .helpers import Error, get_cache_dir, decode_dict, int_to_bigint, \ - bigint_to_int, format_file_size, yes, bin_to_hex, Location + bigint_to_int, format_file_size, yes, bin_to_hex, Location, safe_ns from .locking import Lock from .hashindex import ChunkIndex @@ -461,6 +461,6 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not (self.do_files and stat.S_ISREG(st.st_mode)): return # Entry: Age, inode, size, mtime, chunk ids - mtime_ns = st.st_mtime_ns + mtime_ns = safe_ns(st.st_mtime_ns) self.files[path_hash] = msgpack.packb((0, st.st_ino, st.st_size, int_to_bigint(mtime_ns), ids)) self._newest_mtime = max(self._newest_mtime or 0, mtime_ns) diff --git a/borg/helpers.py b/borg/helpers.py index 299563945..4d5710e8c 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -617,7 +617,7 @@ def timestamp(s): """Convert a --timestamp=s argument to a datetime object""" try: # is it pointing to a file / directory? - ts = os.stat(s).st_mtime + ts = safe_s(os.stat(s).st_mtime) return datetime.utcfromtimestamp(ts) except OSError: # didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support. @@ -728,12 +728,34 @@ def replace_placeholders(text): return format_line(text, data) +# Not too rarely, we get crappy timestamps from the fs, that overflow some computations. +# As they are crap anyway, nothing is lost if we just clamp them to the max valid value. +# msgpack can only pack uint64. datetime is limited to year 9999. +MAX_NS = 18446744073000000000 # less than 2**64 - 1 ns. also less than y9999. +MAX_S = MAX_NS // 1000000000 + + +def safe_s(ts): + if 0 <= ts <= MAX_S: + return ts + elif ts < 0: + return 0 + else: + return MAX_S + + +def safe_ns(ts): + if 0 <= ts <= MAX_NS: + return ts + elif ts < 0: + return 0 + else: + return MAX_NS + + def safe_timestamp(item_timestamp_ns): - try: - return datetime.fromtimestamp(bigint_to_int(item_timestamp_ns) / 1e9) - except OverflowError: - # likely a broken file time and datetime did not want to go beyond year 9999 - return datetime(9999, 12, 31, 23, 59, 59) + t_ns = safe_ns(bigint_to_int(item_timestamp_ns)) + return datetime.fromtimestamp(t_ns / 1e9) def format_time(t): diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 3e5ba18a5..835de3442 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -17,7 +17,8 @@ from ..helpers import Location, format_file_size, format_timedelta, format_line, StableDict, int_to_bigint, bigint_to_int, parse_timestamp, CompressionSpec, ChunkerParams, \ ProgressIndicatorPercent, ProgressIndicatorEndless, parse_pattern, load_exclude_file, load_pattern_file, \ PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern, \ - Buffer + Buffer, safe_ns, safe_s + from . import BaseTestCase, FakeInputs @@ -1115,3 +1116,18 @@ def test_format_line_erroneous(): assert format_line('{invalid}', data) with pytest.raises(PlaceholderError): assert format_line('{}', data) + + +def test_safe_timestamps(): + # ns fit into uint64 + assert safe_ns(2 ** 64) < 2 ** 64 + assert safe_ns(-1) == 0 + # s are so that their ns conversion fits into uint64 + assert safe_s(2 ** 64) * 1000000000 < 2 ** 64 + assert safe_s(-1) == 0 + # datetime won't fall over its y10k problem + beyond_y10k = 2 ** 100 + with pytest.raises(OverflowError): + datetime.utcfromtimestamp(beyond_y10k) + assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31) + assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31)