From da6d1ac538827100fc8a546378fb5fd9bca6ca7d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 4 Apr 2020 18:43:36 +0200 Subject: [PATCH] support msgpack 1.0.0, fixes #5065 our data structures need strict_map_key=False, which is not the default of msgpack 1.0.0. i made it default in our wrapper API. call our wrapper for performance profile creation/conversion also to avoid msgpack compat issues. remove encoding from wrapper api, we do not use it any more. remove raw is True check, we need false for profiles strict_map_key is only supported for msgpack >= 1.0.0. --- setup.py | 2 +- src/borg/archiver.py | 5 +- src/borg/helpers/msgpack.py | 93 +++++++++++++++++++------------------ src/borg/testsuite/cache.py | 2 +- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/setup.py b/setup.py index 8ee14ebf6..52654326c 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ install_requires = [ # we are rather picky about msgpack versions, because a good working msgpack is # very important for borg, see: https://github.com/borgbackup/borg/issues/3753 - 'msgpack >=0.5.6, <=0.6.2', + 'msgpack >=0.5.6, <=1.0.0', # Please note: # using any other version is not supported by borg development and # any feedback related to issues caused by this will be ignored. diff --git a/src/borg/archiver.py b/src/borg/archiver.py index fa2b295c5..6a37d31b6 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2036,7 +2036,7 @@ def do_debug_convert_profile(self, args): """convert Borg profile to Python profile""" import marshal with args.output, args.input: - marshal.dump(msgpack.mp_unpack(args.input, use_list=False, raw=False), args.output) + marshal.dump(msgpack.unpack(args.input, use_list=False, raw=False), args.output) return EXIT_SUCCESS @with_repository(lock=False, manifest=False) @@ -4424,8 +4424,7 @@ def run(self, args): # into a marshal file that can be read by e.g. pyprof2calltree. # For local use it's unnecessary hassle, though, that's why .pyprof makes # it compatible (see above). - # We do not use our msgpack wrapper here, but directly call mp_pack. - msgpack.mp_pack(profiler.stats, fd, use_bin_type=True) + msgpack.pack(profiler.stats, fd, use_bin_type=True) else: return set_ec(func(args)) diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py index 7a83a8449..10d0e2222 100644 --- a/src/borg/helpers/msgpack.py +++ b/src/borg/helpers/msgpack.py @@ -9,13 +9,11 @@ # Packing # ------- # use_bin_type = False is needed to generate the old msgpack format (not msgpack 2.0 spec) as borg always did. -# encoding = None is needed because usage of it is deprecated # unicode_errors = None is needed because usage of it is deprecated # # Unpacking # --------- # raw = True is needed to unpack the old msgpack format to bytes (not str, about the decoding see item.pyx). -# encoding = None is needed because usage of it is deprecated # unicode_errors = None is needed because usage of it is deprecated from msgpack import Packer as mp_Packer @@ -32,6 +30,8 @@ version = mp_version +_post_100 = version >= (1, 0, 0) + class PackException(Exception): """Exception while msgpack packing""" @@ -42,13 +42,11 @@ class UnpackException(Exception): class Packer(mp_Packer): - def __init__(self, *, default=None, encoding=None, unicode_errors=None, + def __init__(self, *, default=None, unicode_errors=None, use_single_float=False, autoreset=True, use_bin_type=False, strict_types=False): - assert use_bin_type is False - assert encoding is None assert unicode_errors is None - super().__init__(default=default, encoding=encoding, unicode_errors=unicode_errors, + super().__init__(default=default, unicode_errors=unicode_errors, use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type, strict_types=strict_types) @@ -59,22 +57,18 @@ def pack(self, obj): raise PackException(e) -def packb(o, *, use_bin_type=False, encoding=None, unicode_errors=None, **kwargs): - assert use_bin_type is False - assert encoding is None +def packb(o, *, use_bin_type=False, unicode_errors=None, **kwargs): assert unicode_errors is None try: - return mp_packb(o, use_bin_type=use_bin_type, encoding=encoding, unicode_errors=unicode_errors, **kwargs) + return mp_packb(o, use_bin_type=use_bin_type, unicode_errors=unicode_errors, **kwargs) except Exception as e: raise PackException(e) -def pack(o, stream, *, use_bin_type=False, encoding=None, unicode_errors=None, **kwargs): - assert use_bin_type is False - assert encoding is None +def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs): assert unicode_errors is None try: - return mp_pack(o, stream, use_bin_type=use_bin_type, encoding=encoding, unicode_errors=unicode_errors, **kwargs) + return mp_pack(o, stream, use_bin_type=use_bin_type, unicode_errors=unicode_errors, **kwargs) except Exception as e: raise PackException(e) @@ -87,25 +81,28 @@ def pack(o, stream, *, use_bin_type=False, encoding=None, unicode_errors=None, * class Unpacker(mp_Unpacker): def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, - encoding=None, unicode_errors=None, max_buffer_size=0, + unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, + strict_map_key=False, max_str_len=2147483647, # 2**32-1 max_bin_len=2147483647, max_array_len=2147483647, max_map_len=2147483647, max_ext_len=2147483647): assert raw is True - assert encoding is None assert unicode_errors is None - super().__init__(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, - object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, - encoding=encoding, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, - ext_hook=ext_hook, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len) + kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, + object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, + unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, + ext_hook=ext_hook, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len) + if _post_100: + kw['strict_map_key'] = strict_map_key + super().__init__(**kw) def unpack(self): try: @@ -126,46 +123,50 @@ def __next__(self): next = __next__ -def unpackb(packed, *, raw=True, encoding=None, unicode_errors=None, +def unpackb(packed, *, raw=True, unicode_errors=None, + strict_map_key=False, max_str_len=2147483647, # 2**32-1 max_bin_len=2147483647, max_array_len=2147483647, max_map_len=2147483647, max_ext_len=2147483647, **kwargs): - assert raw is True - assert encoding is None assert unicode_errors is None try: - return mp_unpackb(packed, raw=raw, encoding=encoding, unicode_errors=unicode_errors, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len, - **kwargs) + kw = dict(raw=raw, unicode_errors=unicode_errors, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len) + kw.update(kwargs) + if _post_100: + kw['strict_map_key'] = strict_map_key + return mp_unpackb(packed, **kw) except Exception as e: raise UnpackException(e) -def unpack(stream, *, raw=True, encoding=None, unicode_errors=None, +def unpack(stream, *, raw=True, unicode_errors=None, + strict_map_key=False, max_str_len=2147483647, # 2**32-1 max_bin_len=2147483647, max_array_len=2147483647, max_map_len=2147483647, max_ext_len=2147483647, **kwargs): - assert raw is True - assert encoding is None assert unicode_errors is None try: - return mp_unpack(stream, raw=raw, encoding=encoding, unicode_errors=unicode_errors, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len, - **kwargs) + kw = dict(raw=raw, unicode_errors=unicode_errors, + max_str_len=max_str_len, + max_bin_len=max_bin_len, + max_array_len=max_array_len, + max_map_len=max_map_len, + max_ext_len=max_ext_len) + kw.update(kwargs) + if _post_100: + kw['strict_map_key'] = strict_map_key + return mp_unpack(stream, **kw) except Exception as e: raise UnpackException(e) @@ -181,7 +182,7 @@ def is_slow_msgpack(): def is_supported_msgpack(): # DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py. import msgpack - return (0, 5, 6) <= msgpack.version <= (0, 6, 2) and \ + return (0, 5, 6) <= msgpack.version <= (1, 0, 0) and \ msgpack.version not in [] # < blacklist bad releases here diff --git a/src/borg/testsuite/cache.py b/src/borg/testsuite/cache.py index 31ebf55af..a0993f74c 100644 --- a/src/borg/testsuite/cache.py +++ b/src/borg/testsuite/cache.py @@ -1,7 +1,7 @@ import io import os.path -from msgpack import packb +from ..helpers.msgpack import packb import pytest