mirror of https://github.com/borgbackup/borg.git
support msgpack 1.0.0, fixes #5065
our data structures need strict_map_key=False, which is not the default of msgpack 1.0.0. i made it default in our wrapper API. call our wrapper for performance profile creation/conversion also to avoid msgpack compat issues. remove encoding from wrapper api, we do not use it any more. remove raw is True check, we need false for profiles strict_map_key is only supported for msgpack >= 1.0.0.
This commit is contained in:
parent
cf8dac55de
commit
da6d1ac538
2
setup.py
2
setup.py
|
@ -71,7 +71,7 @@ on_rtd = os.environ.get('READTHEDOCS')
|
||||||
install_requires = [
|
install_requires = [
|
||||||
# we are rather picky about msgpack versions, because a good working msgpack is
|
# we are rather picky about msgpack versions, because a good working msgpack is
|
||||||
# very important for borg, see: https://github.com/borgbackup/borg/issues/3753
|
# very important for borg, see: https://github.com/borgbackup/borg/issues/3753
|
||||||
'msgpack >=0.5.6, <=0.6.2',
|
'msgpack >=0.5.6, <=1.0.0',
|
||||||
# Please note:
|
# Please note:
|
||||||
# using any other version is not supported by borg development and
|
# using any other version is not supported by borg development and
|
||||||
# any feedback related to issues caused by this will be ignored.
|
# any feedback related to issues caused by this will be ignored.
|
||||||
|
|
|
@ -2036,7 +2036,7 @@ class Archiver:
|
||||||
"""convert Borg profile to Python profile"""
|
"""convert Borg profile to Python profile"""
|
||||||
import marshal
|
import marshal
|
||||||
with args.output, args.input:
|
with args.output, args.input:
|
||||||
marshal.dump(msgpack.mp_unpack(args.input, use_list=False, raw=False), args.output)
|
marshal.dump(msgpack.unpack(args.input, use_list=False, raw=False), args.output)
|
||||||
return EXIT_SUCCESS
|
return EXIT_SUCCESS
|
||||||
|
|
||||||
@with_repository(lock=False, manifest=False)
|
@with_repository(lock=False, manifest=False)
|
||||||
|
@ -4424,8 +4424,7 @@ class Archiver:
|
||||||
# into a marshal file that can be read by e.g. pyprof2calltree.
|
# into a marshal file that can be read by e.g. pyprof2calltree.
|
||||||
# For local use it's unnecessary hassle, though, that's why .pyprof makes
|
# For local use it's unnecessary hassle, though, that's why .pyprof makes
|
||||||
# it compatible (see above).
|
# it compatible (see above).
|
||||||
# We do not use our msgpack wrapper here, but directly call mp_pack.
|
msgpack.pack(profiler.stats, fd, use_bin_type=True)
|
||||||
msgpack.mp_pack(profiler.stats, fd, use_bin_type=True)
|
|
||||||
else:
|
else:
|
||||||
return set_ec(func(args))
|
return set_ec(func(args))
|
||||||
|
|
||||||
|
|
|
@ -9,13 +9,11 @@ from ..constants import * # NOQA
|
||||||
# Packing
|
# Packing
|
||||||
# -------
|
# -------
|
||||||
# use_bin_type = False is needed to generate the old msgpack format (not msgpack 2.0 spec) as borg always did.
|
# use_bin_type = False is needed to generate the old msgpack format (not msgpack 2.0 spec) as borg always did.
|
||||||
# encoding = None is needed because usage of it is deprecated
|
|
||||||
# unicode_errors = None is needed because usage of it is deprecated
|
# unicode_errors = None is needed because usage of it is deprecated
|
||||||
#
|
#
|
||||||
# Unpacking
|
# Unpacking
|
||||||
# ---------
|
# ---------
|
||||||
# raw = True is needed to unpack the old msgpack format to bytes (not str, about the decoding see item.pyx).
|
# raw = True is needed to unpack the old msgpack format to bytes (not str, about the decoding see item.pyx).
|
||||||
# encoding = None is needed because usage of it is deprecated
|
|
||||||
# unicode_errors = None is needed because usage of it is deprecated
|
# unicode_errors = None is needed because usage of it is deprecated
|
||||||
|
|
||||||
from msgpack import Packer as mp_Packer
|
from msgpack import Packer as mp_Packer
|
||||||
|
@ -32,6 +30,8 @@ from msgpack import OutOfData
|
||||||
|
|
||||||
version = mp_version
|
version = mp_version
|
||||||
|
|
||||||
|
_post_100 = version >= (1, 0, 0)
|
||||||
|
|
||||||
|
|
||||||
class PackException(Exception):
|
class PackException(Exception):
|
||||||
"""Exception while msgpack packing"""
|
"""Exception while msgpack packing"""
|
||||||
|
@ -42,13 +42,11 @@ class UnpackException(Exception):
|
||||||
|
|
||||||
|
|
||||||
class Packer(mp_Packer):
|
class Packer(mp_Packer):
|
||||||
def __init__(self, *, default=None, encoding=None, unicode_errors=None,
|
def __init__(self, *, default=None, unicode_errors=None,
|
||||||
use_single_float=False, autoreset=True, use_bin_type=False,
|
use_single_float=False, autoreset=True, use_bin_type=False,
|
||||||
strict_types=False):
|
strict_types=False):
|
||||||
assert use_bin_type is False
|
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
super().__init__(default=default, encoding=encoding, unicode_errors=unicode_errors,
|
super().__init__(default=default, unicode_errors=unicode_errors,
|
||||||
use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type,
|
use_single_float=use_single_float, autoreset=autoreset, use_bin_type=use_bin_type,
|
||||||
strict_types=strict_types)
|
strict_types=strict_types)
|
||||||
|
|
||||||
|
@ -59,22 +57,18 @@ class Packer(mp_Packer):
|
||||||
raise PackException(e)
|
raise PackException(e)
|
||||||
|
|
||||||
|
|
||||||
def packb(o, *, use_bin_type=False, encoding=None, unicode_errors=None, **kwargs):
|
def packb(o, *, use_bin_type=False, unicode_errors=None, **kwargs):
|
||||||
assert use_bin_type is False
|
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
try:
|
try:
|
||||||
return mp_packb(o, use_bin_type=use_bin_type, encoding=encoding, unicode_errors=unicode_errors, **kwargs)
|
return mp_packb(o, use_bin_type=use_bin_type, unicode_errors=unicode_errors, **kwargs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise PackException(e)
|
raise PackException(e)
|
||||||
|
|
||||||
|
|
||||||
def pack(o, stream, *, use_bin_type=False, encoding=None, unicode_errors=None, **kwargs):
|
def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs):
|
||||||
assert use_bin_type is False
|
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
try:
|
try:
|
||||||
return mp_pack(o, stream, use_bin_type=use_bin_type, encoding=encoding, unicode_errors=unicode_errors, **kwargs)
|
return mp_pack(o, stream, use_bin_type=use_bin_type, unicode_errors=unicode_errors, **kwargs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise PackException(e)
|
raise PackException(e)
|
||||||
|
|
||||||
|
@ -87,25 +81,28 @@ def pack(o, stream, *, use_bin_type=False, encoding=None, unicode_errors=None, *
|
||||||
class Unpacker(mp_Unpacker):
|
class Unpacker(mp_Unpacker):
|
||||||
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
|
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
|
||||||
object_hook=None, object_pairs_hook=None, list_hook=None,
|
object_hook=None, object_pairs_hook=None, list_hook=None,
|
||||||
encoding=None, unicode_errors=None, max_buffer_size=0,
|
unicode_errors=None, max_buffer_size=0,
|
||||||
ext_hook=ExtType,
|
ext_hook=ExtType,
|
||||||
|
strict_map_key=False,
|
||||||
max_str_len=2147483647, # 2**32-1
|
max_str_len=2147483647, # 2**32-1
|
||||||
max_bin_len=2147483647,
|
max_bin_len=2147483647,
|
||||||
max_array_len=2147483647,
|
max_array_len=2147483647,
|
||||||
max_map_len=2147483647,
|
max_map_len=2147483647,
|
||||||
max_ext_len=2147483647):
|
max_ext_len=2147483647):
|
||||||
assert raw is True
|
assert raw is True
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
super().__init__(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
|
kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
|
||||||
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
|
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
|
||||||
encoding=encoding, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
|
unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
|
||||||
ext_hook=ext_hook,
|
ext_hook=ext_hook,
|
||||||
max_str_len=max_str_len,
|
max_str_len=max_str_len,
|
||||||
max_bin_len=max_bin_len,
|
max_bin_len=max_bin_len,
|
||||||
max_array_len=max_array_len,
|
max_array_len=max_array_len,
|
||||||
max_map_len=max_map_len,
|
max_map_len=max_map_len,
|
||||||
max_ext_len=max_ext_len)
|
max_ext_len=max_ext_len)
|
||||||
|
if _post_100:
|
||||||
|
kw['strict_map_key'] = strict_map_key
|
||||||
|
super().__init__(**kw)
|
||||||
|
|
||||||
def unpack(self):
|
def unpack(self):
|
||||||
try:
|
try:
|
||||||
|
@ -126,46 +123,50 @@ class Unpacker(mp_Unpacker):
|
||||||
next = __next__
|
next = __next__
|
||||||
|
|
||||||
|
|
||||||
def unpackb(packed, *, raw=True, encoding=None, unicode_errors=None,
|
def unpackb(packed, *, raw=True, unicode_errors=None,
|
||||||
|
strict_map_key=False,
|
||||||
max_str_len=2147483647, # 2**32-1
|
max_str_len=2147483647, # 2**32-1
|
||||||
max_bin_len=2147483647,
|
max_bin_len=2147483647,
|
||||||
max_array_len=2147483647,
|
max_array_len=2147483647,
|
||||||
max_map_len=2147483647,
|
max_map_len=2147483647,
|
||||||
max_ext_len=2147483647,
|
max_ext_len=2147483647,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
assert raw is True
|
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
try:
|
try:
|
||||||
return mp_unpackb(packed, raw=raw, encoding=encoding, unicode_errors=unicode_errors,
|
kw = dict(raw=raw, unicode_errors=unicode_errors,
|
||||||
max_str_len=max_str_len,
|
max_str_len=max_str_len,
|
||||||
max_bin_len=max_bin_len,
|
max_bin_len=max_bin_len,
|
||||||
max_array_len=max_array_len,
|
max_array_len=max_array_len,
|
||||||
max_map_len=max_map_len,
|
max_map_len=max_map_len,
|
||||||
max_ext_len=max_ext_len,
|
max_ext_len=max_ext_len)
|
||||||
**kwargs)
|
kw.update(kwargs)
|
||||||
|
if _post_100:
|
||||||
|
kw['strict_map_key'] = strict_map_key
|
||||||
|
return mp_unpackb(packed, **kw)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise UnpackException(e)
|
raise UnpackException(e)
|
||||||
|
|
||||||
|
|
||||||
def unpack(stream, *, raw=True, encoding=None, unicode_errors=None,
|
def unpack(stream, *, raw=True, unicode_errors=None,
|
||||||
|
strict_map_key=False,
|
||||||
max_str_len=2147483647, # 2**32-1
|
max_str_len=2147483647, # 2**32-1
|
||||||
max_bin_len=2147483647,
|
max_bin_len=2147483647,
|
||||||
max_array_len=2147483647,
|
max_array_len=2147483647,
|
||||||
max_map_len=2147483647,
|
max_map_len=2147483647,
|
||||||
max_ext_len=2147483647,
|
max_ext_len=2147483647,
|
||||||
**kwargs):
|
**kwargs):
|
||||||
assert raw is True
|
|
||||||
assert encoding is None
|
|
||||||
assert unicode_errors is None
|
assert unicode_errors is None
|
||||||
try:
|
try:
|
||||||
return mp_unpack(stream, raw=raw, encoding=encoding, unicode_errors=unicode_errors,
|
kw = dict(raw=raw, unicode_errors=unicode_errors,
|
||||||
max_str_len=max_str_len,
|
max_str_len=max_str_len,
|
||||||
max_bin_len=max_bin_len,
|
max_bin_len=max_bin_len,
|
||||||
max_array_len=max_array_len,
|
max_array_len=max_array_len,
|
||||||
max_map_len=max_map_len,
|
max_map_len=max_map_len,
|
||||||
max_ext_len=max_ext_len,
|
max_ext_len=max_ext_len)
|
||||||
**kwargs)
|
kw.update(kwargs)
|
||||||
|
if _post_100:
|
||||||
|
kw['strict_map_key'] = strict_map_key
|
||||||
|
return mp_unpack(stream, **kw)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise UnpackException(e)
|
raise UnpackException(e)
|
||||||
|
|
||||||
|
@ -181,7 +182,7 @@ def is_slow_msgpack():
|
||||||
def is_supported_msgpack():
|
def is_supported_msgpack():
|
||||||
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
|
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
|
||||||
import msgpack
|
import msgpack
|
||||||
return (0, 5, 6) <= msgpack.version <= (0, 6, 2) and \
|
return (0, 5, 6) <= msgpack.version <= (1, 0, 0) and \
|
||||||
msgpack.version not in [] # < blacklist bad releases here
|
msgpack.version not in [] # < blacklist bad releases here
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
import io
|
import io
|
||||||
import os.path
|
import os.path
|
||||||
|
|
||||||
from msgpack import packb
|
from ..helpers.msgpack import packb
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue