Item: convert timestamps once, get rid of bigint code

src/borg/archiver.py

@@ -366,10 +366,6 @@ def upgrade_item(item):
                 if chunks_healthy is not None:
                     item._dict['chunks_healthy'] = chunks
                 item._dict.pop('source')  # not used for hardlinks any more, replaced by hlid
-            for attr in 'atime', 'ctime', 'mtime', 'birthtime':
-                if attr in item:
-                    ns = getattr(item, attr)  # decode (bigint or Timestamp) --> int ns
-                    setattr(item, attr, ns)  # encode int ns --> msgpack.Timestamp only, no bigint any more
             # make sure we only have desired stuff in the new item. specifically, make sure to get rid of:
             # - 'acl' remnants of bug in attic <= 0.13
             # - 'hardlink_master' (superseded by hlid)

src/borg/helpers/__init__.py

@@ -18,7 +18,7 @@
 from .time import *  # NOQA
 from .yes import *  # NOQA
 
-from .msgpack import is_slow_msgpack, is_supported_msgpack, int_to_bigint, bigint_to_int, get_limited_unpacker
+from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker
 from . import msgpack
 
 # generic mechanism to enable users to invoke workarounds by setting the

src/borg/helpers/msgpack.py

@@ -201,30 +201,11 @@ def get_limited_unpacker(kind):
     return Unpacker(**args)
 
 
-def bigint_to_int(mtime):  # legacy
-    """Convert bytearray to int
-    """
-    if isinstance(mtime, bytes):
-        return int.from_bytes(mtime, 'little', signed=True)
-    return mtime
-
-
-def int_to_bigint(value):  # legacy
-    """Convert integers larger than 64 bits to bytearray
-
-    Smaller integers are left alone
-    """
-    if value.bit_length() > 63:
-        return value.to_bytes((value.bit_length() + 9) // 8, 'little', signed=True)
-    return value
-
-
 def int_to_timestamp(ns):
+    assert isinstance(ns, int)
     return Timestamp.from_unix_nano(ns)
 
 
 def timestamp_to_int(ts):
-    if isinstance(ts, Timestamp):
-        return ts.to_unix_nano()
-    # legacy support note: we need to keep the bigint conversion for compatibility with borg < 1.3 archives.
-    return bigint_to_int(ts)
+    assert isinstance(ts, Timestamp)
+    return ts.to_unix_nano()

src/borg/item.pyx

@@ -4,7 +4,7 @@ from collections import namedtuple
 from .constants import ITEM_KEYS, ARCHIVE_KEYS
 from .helpers import StableDict
 from .helpers import format_file_size
-from .helpers.msgpack import timestamp_to_int, int_to_timestamp
+from .helpers.msgpack import timestamp_to_int, int_to_timestamp, Timestamp
 
 
 cdef extern from "_item.c":
@@ -83,6 +83,17 @@ def fix_tuple_of_str_and_int(v):
     return t
 
 
+def fix_timestamp(v):
+    """make sure v is a Timestamp"""
+    if isinstance(v, Timestamp):
+        return v
+    # legacy support
+    if isinstance(v, bytes):  # was: bigint_to_int()
+        v = int.from_bytes(v, 'little', signed=True)
+    assert isinstance(v, int)
+    return int_to_timestamp(v)
+
+
 def want_bytes(v, *, errors='surrogateescape'):
     """we know that we want bytes and the value should be bytes"""
     # legacy support: it being str can be caused by msgpack unpack decoding old data that was packed with use_bin_type=False
@@ -369,13 +380,16 @@ class Item(PropDict):
             return False
 
     def update_internal(self, d):
-        # legacy support for migration (data from old msgpacks comes in as bytes always, but sometimes we want str)
+        # legacy support for migration (data from old msgpacks comes in as bytes always, but sometimes we want str),
+        # also need to fix old timestamp data types.
         for k, v in list(d.items()):
             k = fix_key(d, k)
             if k in ('path', 'source', 'user', 'group'):
                 v = fix_str_value(d, k)
             if k in ('chunks', 'chunks_healthy'):
                 v = fix_list_of_chunkentries(v)
+            if k in ('atime', 'ctime', 'mtime', 'birthtime'):
+                v = fix_timestamp(v)
             if k in ('acl_access', 'acl_default', 'acl_extended', 'acl_nfs4'):
                 v = fix_bytes_value(d, k)
             # TODO: xattrs

src/borg/testsuite/helpers.py

@@ -22,7 +22,7 @@
 from ..helpers import is_slow_msgpack
 from ..helpers import msgpack
 from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
-from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex
+from ..helpers import StableDict, bin_to_hex
 from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams
 from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless
 from ..helpers import swidth_slice
@@ -38,18 +38,6 @@
 from . import BaseTestCase, FakeInputs
 
 
-class BigIntTestCase(BaseTestCase):
-
-    def test_bigint(self):
-        self.assert_equal(int_to_bigint(0), 0)
-        self.assert_equal(int_to_bigint(2**63-1), 2**63-1)
-        self.assert_equal(int_to_bigint(-2**63+1), -2**63+1)
-        self.assert_equal(int_to_bigint(2**63), b'\x00\x00\x00\x00\x00\x00\x00\x80\x00')
-        self.assert_equal(int_to_bigint(-2**63), b'\x00\x00\x00\x00\x00\x00\x00\x80\xff')
-        self.assert_equal(bigint_to_int(int_to_bigint(-2**70)), -2**70)
-        self.assert_equal(bigint_to_int(int_to_bigint(2**70)), 2**70)
-
-
 def test_bin_to_hex():
     assert bin_to_hex(b'') == ''
     assert bin_to_hex(b'\x00\x01\xff') == '0001ff'