mirror of https://github.com/borgbackup/borg.git
clamp (nano)second values to unproblematic range, fixes #2304
filesystem -> clamp -> archive (create)
This commit is contained in:
parent
d1738ec315
commit
b7a17a6db7
|
@ -33,6 +33,7 @@ from .helpers import format_time, format_timedelta, format_file_size, file_statu
|
|||
from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates
|
||||
from .helpers import StableDict
|
||||
from .helpers import bin_to_hex
|
||||
from .helpers import safe_ns
|
||||
from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi
|
||||
from .helpers import PathPrefixPattern, FnmatchPattern
|
||||
from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec
|
||||
|
@ -784,15 +785,15 @@ Utilization of max. archive size: {csize_max:.0%}
|
|||
mode=st.st_mode,
|
||||
uid=st.st_uid,
|
||||
gid=st.st_gid,
|
||||
mtime=st.st_mtime_ns,
|
||||
mtime=safe_ns(st.st_mtime_ns),
|
||||
)
|
||||
# borg can work with archives only having mtime (older attic archives do not have
|
||||
# atime/ctime). it can be useful to omit atime/ctime, if they change without the
|
||||
# file content changing - e.g. to get better metadata deduplication.
|
||||
if not self.noatime:
|
||||
attrs['atime'] = st.st_atime_ns
|
||||
attrs['atime'] = safe_ns(st.st_atime_ns)
|
||||
if not self.noctime:
|
||||
attrs['ctime'] = st.st_ctime_ns
|
||||
attrs['ctime'] = safe_ns(st.st_ctime_ns)
|
||||
if self.numeric_owner:
|
||||
attrs['user'] = attrs['group'] = None
|
||||
else:
|
||||
|
|
|
@ -17,6 +17,7 @@ from .helpers import Error
|
|||
from .helpers import get_cache_dir, get_security_dir
|
||||
from .helpers import bin_to_hex
|
||||
from .helpers import format_file_size
|
||||
from .helpers import safe_ns
|
||||
from .helpers import yes
|
||||
from .helpers import remove_surrogates
|
||||
from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
|
||||
|
@ -591,6 +592,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
|
|||
def memorize_file(self, path_hash, st, ids):
|
||||
if not (self.do_files and stat.S_ISREG(st.st_mode)):
|
||||
return
|
||||
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=st.st_mtime_ns, chunk_ids=ids)
|
||||
mtime_ns = safe_ns(st.st_mtime_ns)
|
||||
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, mtime=mtime_ns, chunk_ids=ids)
|
||||
self.files[path_hash] = msgpack.packb(entry)
|
||||
self._newest_mtime = max(self._newest_mtime or 0, st.st_mtime_ns)
|
||||
self._newest_mtime = max(self._newest_mtime or 0, mtime_ns)
|
||||
|
|
|
@ -665,7 +665,7 @@ def timestamp(s):
|
|||
"""Convert a --timestamp=s argument to a datetime object"""
|
||||
try:
|
||||
# is it pointing to a file / directory?
|
||||
ts = os.stat(s).st_mtime
|
||||
ts = safe_s(os.stat(s).st_mtime)
|
||||
return datetime.utcfromtimestamp(ts)
|
||||
except OSError:
|
||||
# didn't work, try parsing as timestamp. UTC, no TZ, no microsecs support.
|
||||
|
@ -818,12 +818,34 @@ def SortBySpec(text):
|
|||
return text.replace('timestamp', 'ts')
|
||||
|
||||
|
||||
# Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
|
||||
# As they are crap anyway, nothing is lost if we just clamp them to the max valid value.
|
||||
# msgpack can only pack uint64. datetime is limited to year 9999.
|
||||
MAX_NS = 18446744073000000000 # less than 2**64 - 1 ns. also less than y9999.
|
||||
MAX_S = MAX_NS // 1000000000
|
||||
|
||||
|
||||
def safe_s(ts):
|
||||
if 0 <= ts <= MAX_S:
|
||||
return ts
|
||||
elif ts < 0:
|
||||
return 0
|
||||
else:
|
||||
return MAX_S
|
||||
|
||||
|
||||
def safe_ns(ts):
|
||||
if 0 <= ts <= MAX_NS:
|
||||
return ts
|
||||
elif ts < 0:
|
||||
return 0
|
||||
else:
|
||||
return MAX_NS
|
||||
|
||||
|
||||
def safe_timestamp(item_timestamp_ns):
|
||||
try:
|
||||
return datetime.fromtimestamp(item_timestamp_ns / 1e9)
|
||||
except OverflowError:
|
||||
# likely a broken file time and datetime did not want to go beyond year 9999
|
||||
return datetime(9999, 12, 31, 23, 59, 59)
|
||||
t_ns = safe_ns(item_timestamp_ns)
|
||||
return datetime.fromtimestamp(t_ns / 1e9)
|
||||
|
||||
|
||||
def format_time(t):
|
||||
|
|
|
@ -27,6 +27,7 @@ from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2
|
|||
from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
|
||||
from ..helpers import swidth_slice
|
||||
from ..helpers import chunkit
|
||||
from ..helpers import safe_ns, safe_s
|
||||
|
||||
from . import BaseTestCase, FakeInputs
|
||||
|
||||
|
@ -1221,3 +1222,18 @@ def test_swidth_slice_mixed_characters():
|
|||
string = '나윤a선나윤선나윤선나윤선나윤선'
|
||||
assert swidth_slice(string, 5) == '나윤a'
|
||||
assert swidth_slice(string, 6) == '나윤a'
|
||||
|
||||
|
||||
def test_safe_timestamps():
|
||||
# ns fit into uint64
|
||||
assert safe_ns(2 ** 64) < 2 ** 64
|
||||
assert safe_ns(-1) == 0
|
||||
# s are so that their ns conversion fits into uint64
|
||||
assert safe_s(2 ** 64) * 1000000000 < 2 ** 64
|
||||
assert safe_s(-1) == 0
|
||||
# datetime won't fall over its y10k problem
|
||||
beyond_y10k = 2 ** 100
|
||||
with pytest.raises(OverflowError):
|
||||
datetime.utcfromtimestamp(beyond_y10k)
|
||||
assert datetime.utcfromtimestamp(safe_s(beyond_y10k)) > datetime(2500, 12, 31)
|
||||
assert datetime.utcfromtimestamp(safe_ns(beyond_y10k) / 1000000000) > datetime(2500, 12, 31)
|
||||
|
|
Loading…
Reference in New Issue