mirror of https://github.com/borgbackup/borg.git
transfer: convert timestamps int/bigint -> msgpack.Timestamp, see #2323
Timestamp scales to 64 or 96bit serialization formats, that should be enough for everybody. We use this in archived items and also in the files cache.
This commit is contained in:
parent
e4a97ea8cc
commit
7903dad183
|
@ -360,6 +360,10 @@ class Archiver:
|
|||
if chunks_healthy is not None:
|
||||
item._dict['chunks_healthy'] = chunks
|
||||
item._dict.pop('source') # not used for hardlinks any more, replaced by hlid
|
||||
for attr in 'atime', 'ctime', 'mtime', 'birthtime':
|
||||
if attr in item:
|
||||
ns = getattr(item, attr) # decode (bigint or Timestamp) --> int ns
|
||||
setattr(item, attr, ns) # encode int ns --> msgpack.Timestamp only, no bigint any more
|
||||
item._dict.pop('hardlink_master', None) # not used for hardlinks any more, replaced by hlid
|
||||
item._dict.pop('acl', None) # remove remnants of bug in attic <= 0.13
|
||||
item.get_size(memorize=True) # if not already present: compute+remember size for items with chunks
|
||||
|
|
|
@ -19,7 +19,7 @@ from .helpers import Location
|
|||
from .helpers import Error
|
||||
from .helpers import Manifest
|
||||
from .helpers import get_cache_dir, get_security_dir
|
||||
from .helpers import int_to_bigint, bigint_to_int, bin_to_hex, parse_stringified_list
|
||||
from .helpers import bin_to_hex, parse_stringified_list
|
||||
from .helpers import format_file_size
|
||||
from .helpers import safe_ns
|
||||
from .helpers import yes
|
||||
|
@ -28,6 +28,7 @@ from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
|
|||
from .helpers import set_ec, EXIT_WARNING
|
||||
from .helpers import safe_unlink
|
||||
from .helpers import msgpack
|
||||
from .helpers.msgpack import int_to_timestamp, timestamp_to_int
|
||||
from .item import ArchiveItem, ChunkListEntry
|
||||
from .crypto.key import PlaintextKey
|
||||
from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError
|
||||
|
@ -623,7 +624,7 @@ class LocalCache(CacheStatsMixin):
|
|||
# this is to avoid issues with filesystem snapshots and cmtime granularity.
|
||||
# Also keep files from older backups that have not reached BORG_FILES_CACHE_TTL yet.
|
||||
entry = FileCacheEntry(*msgpack.unpackb(item))
|
||||
if entry.age == 0 and bigint_to_int(entry.cmtime) < self._newest_cmtime or \
|
||||
if entry.age == 0 and timestamp_to_int(entry.cmtime) < self._newest_cmtime or \
|
||||
entry.age > 0 and entry.age < ttl:
|
||||
msgpack.pack((path_hash, entry), fd)
|
||||
entry_count += 1
|
||||
|
@ -1018,10 +1019,10 @@ class LocalCache(CacheStatsMixin):
|
|||
if 'i' in cache_mode and entry.inode != st.st_ino:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file inode number has changed: %r', hashed_path)
|
||||
return True, None
|
||||
if 'c' in cache_mode and bigint_to_int(entry.cmtime) != st.st_ctime_ns:
|
||||
if 'c' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_ctime_ns:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file ctime has changed: %r', hashed_path)
|
||||
return True, None
|
||||
elif 'm' in cache_mode and bigint_to_int(entry.cmtime) != st.st_mtime_ns:
|
||||
elif 'm' in cache_mode and timestamp_to_int(entry.cmtime) != st.st_mtime_ns:
|
||||
files_cache_logger.debug('KNOWN-CHANGED: file mtime has changed: %r', hashed_path)
|
||||
return True, None
|
||||
# we ignored the inode number in the comparison above or it is still same.
|
||||
|
@ -1049,7 +1050,7 @@ class LocalCache(CacheStatsMixin):
|
|||
elif 'm' in cache_mode:
|
||||
cmtime_type = 'mtime'
|
||||
cmtime_ns = safe_ns(st.st_mtime_ns)
|
||||
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_bigint(cmtime_ns), chunk_ids=ids)
|
||||
entry = FileCacheEntry(age=0, inode=st.st_ino, size=st.st_size, cmtime=int_to_timestamp(cmtime_ns), chunk_ids=ids)
|
||||
self.files[path_hash] = msgpack.packb(entry)
|
||||
self._newest_cmtime = max(self._newest_cmtime or 0, cmtime_ns)
|
||||
files_cache_logger.debug('FILES-CACHE-UPDATE: put %r [has %s] <- %r',
|
||||
|
|
|
@ -24,7 +24,7 @@ from msgpack import unpackb as mp_unpackb
|
|||
from msgpack import unpack as mp_unpack
|
||||
from msgpack import version as mp_version
|
||||
|
||||
from msgpack import ExtType
|
||||
from msgpack import ExtType, Timestamp
|
||||
from msgpack import OutOfData
|
||||
|
||||
|
||||
|
@ -164,7 +164,7 @@ def get_limited_unpacker(kind):
|
|||
return Unpacker(**args)
|
||||
|
||||
|
||||
def bigint_to_int(mtime):
|
||||
def bigint_to_int(mtime): # legacy
|
||||
"""Convert bytearray to int
|
||||
"""
|
||||
if isinstance(mtime, bytes):
|
||||
|
@ -172,7 +172,7 @@ def bigint_to_int(mtime):
|
|||
return mtime
|
||||
|
||||
|
||||
def int_to_bigint(value):
|
||||
def int_to_bigint(value): # legacy
|
||||
"""Convert integers larger than 64 bits to bytearray
|
||||
|
||||
Smaller integers are left alone
|
||||
|
@ -180,3 +180,14 @@ def int_to_bigint(value):
|
|||
if value.bit_length() > 63:
|
||||
return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True)
|
||||
return value
|
||||
|
||||
|
||||
def int_to_timestamp(ns):
|
||||
return Timestamp.from_unix_nano(ns)
|
||||
|
||||
|
||||
def timestamp_to_int(ts):
|
||||
if isinstance(ts, Timestamp):
|
||||
return ts.to_unix_nano()
|
||||
# legacy support note: we need to keep the bigint conversion for compatibility with borg < 1.3 archives.
|
||||
return bigint_to_int(ts)
|
||||
|
|
|
@ -19,6 +19,7 @@ logger = create_logger()
|
|||
|
||||
from .errors import Error
|
||||
from .fs import get_keys_dir
|
||||
from .msgpack import Timestamp
|
||||
from .time import OutputTimestamp, format_time, to_localtime, safe_timestamp, safe_s
|
||||
from .. import __version__ as borg_version
|
||||
from .. import __version_tuple__ as borg_version_tuple
|
||||
|
@ -1043,6 +1044,8 @@ def prepare_dump_dict(d):
|
|||
value = decode_tuple(value)
|
||||
elif isinstance(value, bytes):
|
||||
value = decode_bytes(value)
|
||||
elif isinstance(value, Timestamp):
|
||||
value = value.to_unix_nano()
|
||||
if isinstance(key, bytes):
|
||||
key = key.decode()
|
||||
res[key] = value
|
||||
|
|
|
@ -3,9 +3,9 @@ from collections import namedtuple
|
|||
|
||||
from .constants import ITEM_KEYS, ARCHIVE_KEYS
|
||||
from .helpers import safe_encode, safe_decode
|
||||
from .helpers import bigint_to_int, int_to_bigint
|
||||
from .helpers import StableDict
|
||||
from .helpers import format_file_size
|
||||
from .helpers.msgpack import timestamp_to_int, int_to_timestamp
|
||||
|
||||
|
||||
cdef extern from "_item.c":
|
||||
|
@ -171,11 +171,10 @@ class Item(PropDict):
|
|||
rdev = PropDict._make_property('rdev', int)
|
||||
bsdflags = PropDict._make_property('bsdflags', int)
|
||||
|
||||
# note: we need to keep the bigint conversion for compatibility with borg 1.0 archives.
|
||||
atime = PropDict._make_property('atime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
|
||||
ctime = PropDict._make_property('ctime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
|
||||
mtime = PropDict._make_property('mtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
|
||||
birthtime = PropDict._make_property('birthtime', int, 'bigint', encode=int_to_bigint, decode=bigint_to_int)
|
||||
atime = PropDict._make_property('atime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int)
|
||||
ctime = PropDict._make_property('ctime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int)
|
||||
mtime = PropDict._make_property('mtime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int)
|
||||
birthtime = PropDict._make_property('birthtime', int, 'int (ns)', encode=int_to_timestamp, decode=timestamp_to_int)
|
||||
|
||||
# size is only present for items with a chunk list and then it is sum(chunk_sizes)
|
||||
# compatibility note: this is a new feature, in old archives size will be missing.
|
||||
|
|
|
@ -3,6 +3,7 @@ import pytest
|
|||
from ..cache import ChunkListEntry
|
||||
from ..item import Item
|
||||
from ..helpers import StableDict
|
||||
from ..helpers.msgpack import Timestamp
|
||||
|
||||
|
||||
def test_item_empty():
|
||||
|
@ -77,15 +78,15 @@ def test_item_int_property():
|
|||
item.mode = "invalid"
|
||||
|
||||
|
||||
def test_item_bigint_property():
|
||||
def test_item_mptimestamp_property():
|
||||
item = Item()
|
||||
small, big = 42, 2 ** 65
|
||||
item.atime = small
|
||||
assert item.atime == small
|
||||
assert item.as_dict() == {'atime': small}
|
||||
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(small)}
|
||||
item.atime = big
|
||||
assert item.atime == big
|
||||
assert item.as_dict() == {'atime': b'\0' * 8 + b'\x02'}
|
||||
assert item.as_dict() == {'atime': Timestamp.from_unix_nano(big)}
|
||||
|
||||
|
||||
def test_item_user_group_none():
|
||||
|
|
Loading…
Reference in New Issue