mirror of https://github.com/borgbackup/borg.git
borg rpc: use limited msgpack.Unpacker, fixes #2139
we do not trust the remote, so we are careful unpacking its responses. the remote could return manipulated msgpack data that announces e.g. a huge array or map or string. the local would then need to allocate huge amounts of RAM in expectation of that data (no matter whether really that much is coming or not). by using limits in the Unpacker, a ValueError will be raised if unexpected amounts of data shall get unpacked. memory DoS will be avoided. # Conflicts: # borg/archiver.py # src/borg/archive.py # src/borg/remote.py # src/borg/repository.py
This commit is contained in:
parent
26b4a3938b
commit
29613938d5
|
@ -23,7 +23,7 @@ from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex
|
|||
from .platform import acl_get, acl_set
|
||||
from .chunker import Chunker
|
||||
from .hashindex import ChunkIndex
|
||||
from .repository import Repository
|
||||
from .repository import Repository, LIST_SCAN_LIMIT
|
||||
|
||||
import msgpack
|
||||
|
||||
|
@ -882,7 +882,7 @@ class ArchiveChecker:
|
|||
self.chunks = ChunkIndex(capacity)
|
||||
marker = None
|
||||
while True:
|
||||
result = self.repository.list(limit=10000, marker=marker)
|
||||
result = self.repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
|
||||
if not result:
|
||||
break
|
||||
marker = result[-1]
|
||||
|
|
|
@ -29,7 +29,7 @@ from .logger import create_logger, setup_logging
|
|||
logger = create_logger()
|
||||
from .compress import Compressor
|
||||
from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
|
||||
from .repository import Repository
|
||||
from .repository import Repository, LIST_SCAN_LIMIT
|
||||
from .cache import Cache
|
||||
from .key import key_creator, tam_required_file, tam_required, RepoKey, PassphraseKey
|
||||
from .keymanager import KeyManager
|
||||
|
@ -813,7 +813,7 @@ class Archiver:
|
|||
marker = None
|
||||
i = 0
|
||||
while True:
|
||||
result = repository.list(limit=10000, marker=marker)
|
||||
result = repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
|
||||
if not result:
|
||||
break
|
||||
marker = result[-1]
|
||||
|
|
|
@ -15,7 +15,7 @@ from . import __version__
|
|||
from .helpers import Error, IntegrityError, sysinfo
|
||||
from .helpers import replace_placeholders
|
||||
from .helpers import bin_to_hex
|
||||
from .repository import Repository
|
||||
from .repository import Repository, LIST_SCAN_LIMIT, MAX_OBJECT_SIZE
|
||||
from .logger import create_logger
|
||||
|
||||
import msgpack
|
||||
|
@ -46,6 +46,27 @@ def os_write(fd, data):
|
|||
return amount
|
||||
|
||||
|
||||
def get_limited_unpacker(kind):
|
||||
"""return a limited Unpacker because we should not trust msgpack data received from remote"""
|
||||
args = dict(use_list=False, # return tuples, not lists
|
||||
max_bin_len=0, # not used
|
||||
max_ext_len=0, # not used
|
||||
max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
|
||||
max_str_len=MAX_OBJECT_SIZE, # a chunk or other repo object
|
||||
)
|
||||
if kind == 'server':
|
||||
args.update(dict(max_array_len=100, # misc. cmd tuples
|
||||
max_map_len=100, # misc. cmd dicts
|
||||
))
|
||||
elif kind == 'client':
|
||||
args.update(dict(max_array_len=LIST_SCAN_LIMIT, # result list from repo.list() / .scan()
|
||||
max_map_len=100, # misc. result dicts
|
||||
))
|
||||
else:
|
||||
raise ValueError('kind must be "server" or "client"')
|
||||
return msgpack.Unpacker(**args)
|
||||
|
||||
|
||||
class ConnectionClosed(Error):
|
||||
"""Connection closed by remote host"""
|
||||
|
||||
|
@ -115,7 +136,7 @@ class RepositoryServer: # pragma: no cover
|
|||
# Make stderr blocking
|
||||
fl = fcntl.fcntl(stderr_fd, fcntl.F_GETFL)
|
||||
fcntl.fcntl(stderr_fd, fcntl.F_SETFL, fl & ~os.O_NONBLOCK)
|
||||
unpacker = msgpack.Unpacker(use_list=False)
|
||||
unpacker = get_limited_unpacker('server')
|
||||
while True:
|
||||
r, w, es = select.select([stdin_fd], [], [], 10)
|
||||
if r:
|
||||
|
@ -205,7 +226,7 @@ class RemoteRepository:
|
|||
self.cache = {}
|
||||
self.ignore_responses = set()
|
||||
self.responses = {}
|
||||
self.unpacker = msgpack.Unpacker(use_list=False)
|
||||
self.unpacker = get_limited_unpacker('client')
|
||||
self.p = None
|
||||
testing = location.host == '__testsuite__'
|
||||
borg_cmd = self.borg_cmd(args, testing)
|
||||
|
|
|
@ -26,6 +26,8 @@ TAG_PUT = 0
|
|||
TAG_DELETE = 1
|
||||
TAG_COMMIT = 2
|
||||
|
||||
LIST_SCAN_LIMIT = 10000 # repo.list() / .scan() result count limit the borg client uses
|
||||
|
||||
|
||||
class Repository:
|
||||
"""Filesystem based transactional key value store
|
||||
|
|
Loading…
Reference in New Issue