mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-02-22 06:01:54 +00:00
Code Issues Releases Wiki Activity

Merge pull request #6341 from ThomasWaldmann/require-msgpack10

Browse source 
Require msgpack 1.0.3
This commit is contained in:
TW 2022-02-26 00:51:53 +01:00 committed by GitHub
commit cf4a1abb87
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
setup.py
View file

@ -66,7 +66,7 @@
install_requires = [ install_requires = [
# we are rather picky about msgpack versions, because a good working msgpack is # we are rather picky about msgpack versions, because a good working msgpack is
# very important for borg, see: https://github.com/borgbackup/borg/issues/3753 # very important for borg, see: https://github.com/borgbackup/borg/issues/3753
'msgpack >=0.5.6, <=1.0.3, !=1.0.1', 'msgpack >=1.0.3, <=1.0.3',
# Please note: # Please note:
# using any other version is not supported by borg development and # using any other version is not supported by borg development and
# any feedback related to issues caused by this will be ignored. # any feedback related to issues caused by this will be ignored.

79
src/borg/helpers/msgpack.py
View file

@ -30,8 +30,6 @@
version = mp_version version = mp_version
_post_100 = version >= (1, 0, 0)
class PackException(Exception): class PackException(Exception):
"""Exception while msgpack packing""" """Exception while msgpack packing"""
@ -73,35 +71,19 @@ def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs):
raise PackException(e) raise PackException(e)
# Note: after requiring msgpack >= 0.6.1 we can remove the max_*_len args and
# rely on msgpack auto-computing DoS-safe max values from len(data) for
# unpack(data) or from max_buffer_len for Unpacker(max_buffer_len=N).
# maybe we can also use that to simplify get_limited_unpacker().
class Unpacker(mp_Unpacker): class Unpacker(mp_Unpacker):
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True, def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
object_hook=None, object_pairs_hook=None, list_hook=None, object_hook=None, object_pairs_hook=None, list_hook=None,
unicode_errors=None, max_buffer_size=0, unicode_errors=None, max_buffer_size=0,
ext_hook=ExtType, ext_hook=ExtType,
strict_map_key=False, strict_map_key=False):
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647):
assert raw is True assert raw is True
assert unicode_errors is None assert unicode_errors is None
kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
ext_hook=ext_hook, ext_hook=ext_hook,
max_str_len=max_str_len, strict_map_key=strict_map_key)
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
if _post_100:
kw['strict_map_key'] = strict_map_key
super().__init__(**kw) super().__init__(**kw)
def unpack(self): def unpack(self):
@ -125,23 +107,12 @@ def __next__(self):
def unpackb(packed, *, raw=True, unicode_errors=None, def unpackb(packed, *, raw=True, unicode_errors=None,
strict_map_key=False, strict_map_key=False,
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647,
**kwargs): **kwargs):
assert unicode_errors is None assert unicode_errors is None
try: try:
kw = dict(raw=raw, unicode_errors=unicode_errors, kw = dict(raw=raw, unicode_errors=unicode_errors,
max_str_len=max_str_len, strict_map_key=strict_map_key)
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
kw.update(kwargs) kw.update(kwargs)
if _post_100:
kw['strict_map_key'] = strict_map_key
return mp_unpackb(packed, **kw) return mp_unpackb(packed, **kw)
except Exception as e: except Exception as e:
raise UnpackException(e) raise UnpackException(e)
@ -149,23 +120,12 @@ def unpackb(packed, *, raw=True, unicode_errors=None,
def unpack(stream, *, raw=True, unicode_errors=None, def unpack(stream, *, raw=True, unicode_errors=None,
strict_map_key=False, strict_map_key=False,
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647,
**kwargs): **kwargs):
assert unicode_errors is None assert unicode_errors is None
try: try:
kw = dict(raw=raw, unicode_errors=unicode_errors, kw = dict(raw=raw, unicode_errors=unicode_errors,
max_str_len=max_str_len, strict_map_key=strict_map_key)
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
kw.update(kwargs) kw.update(kwargs)
if _post_100:
kw['strict_map_key'] = strict_map_key
return mp_unpack(stream, **kw) return mp_unpack(stream, **kw)
except Exception as e: except Exception as e:
raise UnpackException(e) raise UnpackException(e)
@ -182,38 +142,21 @@ def is_slow_msgpack():
def is_supported_msgpack(): def is_supported_msgpack():
# DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py. # DO NOT CHANGE OR REMOVE! See also requirements and comments in setup.py.
import msgpack import msgpack
return (0, 5, 6) <= msgpack.version <= (1, 0, 3) and \ return (1, 0, 3) <= msgpack.version <= (1, 0, 3) and \
msgpack.version not in [(1, 0, 1), ] # < add bad releases here to deny list msgpack.version not in [] # < add bad releases here to deny list
def get_limited_unpacker(kind): def get_limited_unpacker(kind):
"""return a limited Unpacker because we should not trust msgpack data received from remote""" """return a limited Unpacker because we should not trust msgpack data received from remote"""
# Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for
# unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N).
args = dict(use_list=False, # return tuples, not lists args = dict(use_list=False, # return tuples, not lists
max_bin_len=0, # not used
max_ext_len=0, # not used
max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE), max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
max_str_len=MAX_OBJECT_SIZE, # a chunk or other repo object
) )
if kind == 'server': if kind in ('server', 'client'):
args.update(dict(max_array_len=100, # misc. cmd tuples pass # nothing special
max_map_len=100, # misc. cmd dicts elif kind in ('manifest', 'key'):
))
elif kind == 'client':
args.update(dict(max_array_len=LIST_SCAN_LIMIT, # result list from repo.list() / .scan()
max_map_len=100, # misc. result dicts
))
elif kind == 'manifest':
args.update(dict(use_list=True, # default value args.update(dict(use_list=True, # default value
max_array_len=100, # ITEM_KEYS ~= 22
max_map_len=MAX_ARCHIVES, # list of archives
max_str_len=255, # archive name
object_hook=StableDict,
))
elif kind == 'key':
args.update(dict(use_list=True, # default value
max_array_len=0, # not used
max_map_len=10, # EncryptedKey dict
max_str_len=4000, # inner key data
object_hook=StableDict, object_hook=StableDict,
)) ))
else: else: