mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2024-12-29 11:16:43 +00:00
Code Issues Releases Wiki Activity

Merge pull request #1418 from textshell/fix/preload

Browse source 
Cleanup prefetch code in RemoteRepository
This commit is contained in:
enkore 2016-10-07 09:49:16 +02:00 committed by GitHub
commit 8567fcfe58
1 changed files with 14 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
src/borg/remote.py
View file

@ -216,7 +216,7 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock
self.preload_ids = [] self.preload_ids = []
self.msgid = 0 self.msgid = 0
self.to_send = b'' self.to_send = b''
self.cache = {} self.chunkid_to_msgids = {}
self.ignore_responses = set() self.ignore_responses = set()
self.responses = {} self.responses = {}
self.ratelimit = SleepingBandwidthLimiter(args.remote_ratelimit * 1024 if args and args.remote_ratelimit else 0) self.ratelimit = SleepingBandwidthLimiter(args.remote_ratelimit * 1024 if args and args.remote_ratelimit else 0)
@ -350,10 +350,10 @@ def call_many(self, cmd, calls, wait=True, is_preloaded=False):
if not calls: if not calls:
return return
def fetch_from_cache(args): def pop_preload_msgid(chunkid):
msgid = self.cache[args].pop(0) msgid = self.chunkid_to_msgids[chunkid].pop(0)
if not self.cache[args]: if not self.chunkid_to_msgids[chunkid]:
del self.cache[args] del self.chunkid_to_msgids[chunkid]
return msgid return msgid
def handle_error(error, res): def handle_error(error, res):
@ -424,21 +424,23 @@ def handle_error(error, res):
while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT:
if calls: if calls:
if is_preloaded: if is_preloaded:
if calls[0] in self.cache: assert cmd == "get", "is_preload is only supported for 'get'"
waiting_for.append(fetch_from_cache(calls.pop(0))) if calls[0][0] in self.chunkid_to_msgids:
waiting_for.append(pop_preload_msgid(calls.pop(0)[0]))
else: else:
args = calls.pop(0) args = calls.pop(0)
if cmd == 'get' and args in self.cache: if cmd == 'get' and args[0] in self.chunkid_to_msgids:
waiting_for.append(fetch_from_cache(args)) waiting_for.append(pop_preload_msgid(args[0]))
else: else:
self.msgid += 1 self.msgid += 1
waiting_for.append(self.msgid) waiting_for.append(self.msgid)
self.to_send = msgpack.packb((1, self.msgid, cmd, args)) self.to_send = msgpack.packb((1, self.msgid, cmd, args))
if not self.to_send and self.preload_ids: if not self.to_send and self.preload_ids:
args = (self.preload_ids.pop(0),) chunk_id = self.preload_ids.pop(0)
args = (chunk_id,)
self.msgid += 1 self.msgid += 1
self.cache.setdefault(args, []).append(self.msgid) self.chunkid_to_msgids.setdefault(chunk_id, []).append(self.msgid)
self.to_send = msgpack.packb((1, self.msgid, cmd, args)) self.to_send = msgpack.packb((1, self.msgid, 'get', args))
if self.to_send: if self.to_send:
try: try: