mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-02-28 08:46:56 +00:00
Code Issues Releases Wiki Activity

Pipeline improvements and minor code cleanup.

Browse source
This commit is contained in:
Jonas Borgström 2012-11-28 21:17:00 +01:00
parent 514ab1f386
commit 7245575fa2
2 changed files with 35 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
darc/archive.py
View file

@ -1,7 +1,7 @@
from __future__ import with_statement from __future__ import with_statement
from datetime import datetime, timedelta from datetime import datetime, timedelta
from getpass import getuser from getpass import getuser
from itertools import izip from itertools import izip_longest
import msgpack import msgpack
import os import os
import socket import socket
@ -29,6 +29,7 @@ def __init__(self, unpacker, filter):
self.unpacker = iter(unpacker) self.unpacker = iter(unpacker)
self.filter = filter self.filter = filter
self.stack = [] self.stack = []
self.peeks = 0
self._peek = None self._peek = None
self._peek_iter = None self._peek_iter = None
global foo global foo
@ -39,9 +40,12 @@ def __iter__(self):
def next(self): def next(self):
if self.stack: if self.stack:
return self.stack.pop(0) item = self.stack.pop(0)
else:
self._peek = None self._peek = None
return self.get_next() item = self.get_next()
self.peeks = max(0, self.peeks - len(item.get('chunks', [])))
return item
def get_next(self): def get_next(self):
next = self.unpacker.next() next = self.unpacker.next()
@ -52,7 +56,7 @@ def get_next(self):
def peek(self): def peek(self):
while True: while True:
while not self._peek or not self._peek_iter: while not self._peek or not self._peek_iter:
if len(self.stack) > 100: if self.peeks > 100:
raise StopIteration raise StopIteration
self._peek = self.get_next() self._peek = self.get_next()
self.stack.append(self._peek) self.stack.append(self._peek)
@ -61,7 +65,9 @@ def peek(self):
else: else:
self._peek_iter = None self._peek_iter = None
try: try:
return self._peek_iter.next() item = self._peek_iter.next()
self.peeks += 1
return item
except StopIteration: except StopIteration:
self._peek = None self._peek = None
@ -186,25 +192,26 @@ def save(self, name=None):
self.cache.commit() self.cache.commit()
def calc_stats(self, cache): def calc_stats(self, cache):
def add(id):
count, size, csize = self.cache.chunks[id]
stats.update(size, csize, count == 1)
self.cache.chunks[id] = count - 1, size, csize
# This function is a bit evil since it abuses the cache to calculate # This function is a bit evil since it abuses the cache to calculate
# the stats. The cache transaction must be rolled back afterwards # the stats. The cache transaction must be rolled back afterwards
unpacker = msgpack.Unpacker() unpacker = msgpack.Unpacker()
cache.begin_txn() cache.begin_txn()
stats = Statistics() stats = Statistics()
for id in self.metadata['items']: add(self.id)
unpacker.feed(self.key.decrypt(id, self.store.get(id))) for id, chunk in izip_longest(self.metadata['items'], self.store.get_many(self.metadata['items'])):
unpacker.feed(self.key.decrypt(id, chunk))
for item in unpacker: for item in unpacker:
try: try:
for id, size, csize in item['chunks']: for id, size, csize in item['chunks']:
count, _, _ = self.cache.chunks[id] add(id)
stats.update(size, csize, count == 1)
stats.nfiles += 1 stats.nfiles += 1
self.cache.chunks[id] = count - 1, size, csize
except KeyError: except KeyError:
pass pass
count, size, csize = self.cache.chunks[id] add(id)
stats.update(size, csize, count == 1)
self.cache.chunks[id] = count - 1, size, csize
cache.rollback() cache.rollback()
return stats return stats
@ -237,7 +244,7 @@ def extract_item(self, item, dest=None, start_cb=None, restore_attrs=True, peek=
fd = open(path, 'wb') fd = open(path, 'wb')
start_cb(item) start_cb(item)
ids = [id for id, size, csize in item['chunks']] ids = [id for id, size, csize in item['chunks']]
for id, chunk in izip(ids, self.store.get_many(ids, peek)): for id, chunk in izip_longest(ids, self.store.get_many(ids, peek)):
data = self.key.decrypt(id, chunk) data = self.key.decrypt(id, chunk)
fd.write(data) fd.write(data)
fd.close() fd.close()
@ -296,8 +303,7 @@ def verify_file(self, item, start, result, peek=None):
start(item) start(item)
ids = [id for id, size, csize in item['chunks']] ids = [id for id, size, csize in item['chunks']]
try: try:
for id, chunk in izip(ids, self.store.get_many(ids, peek)): for id, chunk in izip_longest(ids, self.store.get_many(ids, peek)):
if chunk:
self.key.decrypt(id, chunk) self.key.decrypt(id, chunk)
except Exception, e: except Exception, e:
result(item, False) result(item, False)

21
darc/remote.py
View file

@ -75,7 +75,7 @@ def __init__(self, location, create=False):
self.cache = LRUCache(200) self.cache = LRUCache(200)
self.to_send = '' self.to_send = ''
self.extra = {} self.extra = {}
self.pending_cache = {} self.pending = {}
self.unpacker = msgpack.Unpacker() self.unpacker = msgpack.Unpacker()
self.msgid = 0 self.msgid = 0
self.received_msgid = 0 self.received_msgid = 0
@ -106,13 +106,10 @@ def _read(self):
self.received_msgid = msgid self.received_msgid = msgid
if error: if error:
raise self.RPCError(error) raise self.RPCError(error)
if msgid in self.pending_cache: args = self.pending.pop(msgid)
args = self.pending_cache.pop(msgid)
self.cache[args] = msgid, res self.cache[args] = msgid, res
else: for args, resp in self.extra.pop(msgid, []):
print 'unknown response' to_yield.append(resp or self.cache[args][1])
for args in self.extra.pop(msgid, []):
to_yield.append(self.cache[args][1])
for res in to_yield: for res in to_yield:
yield res yield res
@ -127,12 +124,12 @@ def gen_request(self, cmd, argsv):
if not args in self.cache: if not args in self.cache:
self.msgid += 1 self.msgid += 1
msgid = self.msgid msgid = self.msgid
self.pending_cache[msgid] = args self.pending[msgid] = args
self.cache[args] = msgid, None self.cache[args] = msgid, None
data.append(msgpack.packb((1, msgid, cmd, args))) data.append(msgpack.packb((1, msgid, cmd, args)))
msgid, resp = self.cache[args] msgid, resp = self.cache[args]
m = max(m, msgid) m = max(m, msgid)
self.extra.setdefault(m, []).append(args) self.extra.setdefault(m, []).append((args, resp))
return ''.join(data) return ''.join(data)
def gen_cache_requests(self, cmd, peek): def gen_cache_requests(self, cmd, peek):
@ -146,7 +143,7 @@ def gen_cache_requests(self, cmd, peek):
continue continue
self.msgid += 1 self.msgid += 1
msgid = self.msgid msgid = self.msgid
self.pending_cache[msgid] = args self.pending[msgid] = args
self.cache[args] = msgid, None self.cache[args] = msgid, None
data.append(msgpack.packb((1, msgid, cmd, args))) data.append(msgpack.packb((1, msgid, cmd, args)))
return ''.join(data) return ''.join(data)
@ -156,9 +153,9 @@ def call_multi(self, cmd, argsv, wait=True, peek=None):
left = len(argsv) left = len(argsv)
data = self.gen_request(cmd, argsv) data = self.gen_request(cmd, argsv)
self.to_send += data self.to_send += data
for args in self.extra.pop(self.received_msgid, []): for args, resp in self.extra.pop(self.received_msgid, []):
left -= 1 left -= 1
yield self.cache[args][1] yield resp or self.cache[args][1]
while left: while left:
r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1) r, w, x = select.select(self.r_fds, w_fds, self.x_fds, 1)
if x: if x: