From e9605d67adebe7a5f01b66bcaf9ea35709e26b15 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Fri, 5 Aug 2016 21:45:12 +0200 Subject: [PATCH 1/3] RemoteReposity: prefetch can only be 'get'. --- src/borg/remote.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index d48fa9495..b45fb9ec8 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -424,6 +424,7 @@ def handle_error(error, res): while not self.to_send and (calls or self.preload_ids) and len(waiting_for) < MAX_INFLIGHT: if calls: if is_preloaded: + assert cmd == "get", "is_preload is only supported for 'get'" if calls[0] in self.cache: waiting_for.append(fetch_from_cache(calls.pop(0))) else: @@ -438,7 +439,7 @@ def handle_error(error, res): args = (self.preload_ids.pop(0),) self.msgid += 1 self.cache.setdefault(args, []).append(self.msgid) - self.to_send = msgpack.packb((1, self.msgid, cmd, args)) + self.to_send = msgpack.packb((1, self.msgid, 'get', args)) if self.to_send: try: From 2608a5620a3cb817271518d71977a8dc9c14b0dc Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Fri, 5 Aug 2016 21:48:23 +0200 Subject: [PATCH 2/3] RemoteRepository: Always store chunk ids in cache instead of rpc argument encoding of get request. --- src/borg/remote.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index b45fb9ec8..029933672 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -425,20 +425,21 @@ def handle_error(error, res): if calls: if is_preloaded: assert cmd == "get", "is_preload is only supported for 'get'" - if calls[0] in self.cache: - waiting_for.append(fetch_from_cache(calls.pop(0))) + if calls[0][0] in self.cache: + waiting_for.append(fetch_from_cache(calls.pop(0)[0])) else: args = calls.pop(0) - if cmd == 'get' and args in self.cache: - waiting_for.append(fetch_from_cache(args)) + if cmd == 'get' and args[0] in self.cache: + waiting_for.append(fetch_from_cache(args[0])) else: self.msgid += 1 waiting_for.append(self.msgid) self.to_send = msgpack.packb((1, self.msgid, cmd, args)) if not self.to_send and self.preload_ids: - args = (self.preload_ids.pop(0),) + chunk_id = self.preload_ids.pop(0) + args = (chunk_id,) self.msgid += 1 - self.cache.setdefault(args, []).append(self.msgid) + self.cache.setdefault(chunk_id, []).append(self.msgid) self.to_send = msgpack.packb((1, self.msgid, 'get', args)) if self.to_send: From 02557f16b05fb733509e906b9a4224dcb2ca755b Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Fri, 5 Aug 2016 21:51:01 +0200 Subject: [PATCH 3/3] RemoteRepository: Rename cache to chunkid_to_msgids. Also fetch_from_cache to pop_preload_msgid. --- src/borg/remote.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index 029933672..824161f04 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -216,7 +216,7 @@ def __init__(self, location, create=False, exclusive=False, lock_wait=None, lock self.preload_ids = [] self.msgid = 0 self.to_send = b'' - self.cache = {} + self.chunkid_to_msgids = {} self.ignore_responses = set() self.responses = {} self.ratelimit = SleepingBandwidthLimiter(args.remote_ratelimit * 1024 if args and args.remote_ratelimit else 0) @@ -350,10 +350,10 @@ def call_many(self, cmd, calls, wait=True, is_preloaded=False): if not calls: return - def fetch_from_cache(args): - msgid = self.cache[args].pop(0) - if not self.cache[args]: - del self.cache[args] + def pop_preload_msgid(chunkid): + msgid = self.chunkid_to_msgids[chunkid].pop(0) + if not self.chunkid_to_msgids[chunkid]: + del self.chunkid_to_msgids[chunkid] return msgid def handle_error(error, res): @@ -425,12 +425,12 @@ def handle_error(error, res): if calls: if is_preloaded: assert cmd == "get", "is_preload is only supported for 'get'" - if calls[0][0] in self.cache: - waiting_for.append(fetch_from_cache(calls.pop(0)[0])) + if calls[0][0] in self.chunkid_to_msgids: + waiting_for.append(pop_preload_msgid(calls.pop(0)[0])) else: args = calls.pop(0) - if cmd == 'get' and args[0] in self.cache: - waiting_for.append(fetch_from_cache(args[0])) + if cmd == 'get' and args[0] in self.chunkid_to_msgids: + waiting_for.append(pop_preload_msgid(args[0])) else: self.msgid += 1 waiting_for.append(self.msgid) @@ -439,7 +439,7 @@ def handle_error(error, res): chunk_id = self.preload_ids.pop(0) args = (chunk_id,) self.msgid += 1 - self.cache.setdefault(chunk_id, []).append(self.msgid) + self.chunkid_to_msgids.setdefault(chunk_id, []).append(self.msgid) self.to_send = msgpack.packb((1, self.msgid, 'get', args)) if self.to_send: