borg/dedupestore/cache.py

110 lines
3.5 KiB
Python
Raw Normal View History

2010-03-06 17:25:35 +00:00
import cPickle
import hashlib
import os
import sys
import zlib
from chunkifier import checksum
from store import Store, NS_ARCHIVES, NS_CHUNKS
class Cache(object):
"""Client Side cache
"""
def __init__(self, store):
self.store = store
self.path = os.path.join(os.path.expanduser('~'), '.dedupestore', 'cache',
'%s.cache' % self.store.uuid)
self.tid = -1
self.open()
if self.tid != self.store.tid:
self.init()
def open(self):
if not os.path.exists(self.path):
return
print 'Loading cache: ', self.path, '...'
data = cPickle.loads(zlib.decompress(open(self.path, 'rb').read()))
if data['uuid'] != self.store.uuid:
print >> sys.stderr, 'Cache UUID mismatch'
return
self.chunkmap = data['chunkmap']
self.summap = data['summap']
self.archives = data['archives']
self.tid = data['tid']
print 'done'
def init(self):
"""Initializes cache by fetching and reading all archive indicies
"""
self.summap = {}
self.chunkmap = {}
self.archives = []
self.tid = self.store.tid
if self.store.tid == 0:
return
print 'Recreating cache...'
for id in self.store.list(NS_ARCHIVES):
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, id)))
self.archives.append(archive['name'])
for id, sum, csize, osize in archive['chunks']:
if self.seen_chunk(id):
self.chunk_incref(id)
else:
self.init_chunk(id, sum, csize, osize)
2010-03-06 17:25:35 +00:00
print 'done'
def save(self):
assert self.store.state == Store.OPEN
print 'saving cache'
2010-04-18 20:08:12 +00:00
data = {'uuid': self.store.uuid,
2010-03-06 17:25:35 +00:00
'chunkmap': self.chunkmap, 'summap': self.summap,
'tid': self.store.tid, 'archives': self.archives}
print 'Saving cache as:', self.path
cachedir = os.path.dirname(self.path)
if not os.path.exists(cachedir):
os.makedirs(cachedir)
with open(self.path, 'wb') as fd:
fd.write(zlib.compress(cPickle.dumps(data)))
print 'done'
def add_chunk(self, data):
sum = checksum(data)
osize = len(data)
2010-03-06 17:25:35 +00:00
data = zlib.compress(data)
id = hashlib.sha1(data).digest()
if self.seen_chunk(id):
return self.chunk_incref(id)
csize = len(data)
self.store.put(NS_CHUNKS, id, data)
return self.init_chunk(id, sum, csize, osize)
def init_chunk(self, id, sum, csize, osize):
self.chunkmap[id] = (1, sum, osize, csize)
self.summap.setdefault(sum, 1)
return id, sum, csize, osize
2010-03-06 17:25:35 +00:00
def seen_chunk(self, id):
return id in self.chunkmap
2010-03-06 17:25:35 +00:00
def chunk_incref(self, id):
count, sum, csize, osize = self.chunkmap[id]
self.chunkmap[id] = (count + 1, sum, osize, csize)
2010-03-06 17:25:35 +00:00
self.summap[sum] += 1
return id, sum, csize, osize
2010-03-06 17:25:35 +00:00
def chunk_decref(self, id):
count, sum, csize, osize = self.chunkmap[id]
2010-04-18 20:08:12 +00:00
sumcount = self.summap[sum]
if sumcount == 1:
2010-03-06 17:25:35 +00:00
del self.summap[sum]
else:
self.summap[sum] = sumcount - 1
if count == 1:
2010-03-06 17:25:35 +00:00
del self.chunkmap[id]
print 'deleting chunk: ', id.encode('hex')
self.store.delete(NS_CHUNKS, id)
else:
self.chunkmap[id] = (count - 1, sum, csize, osize)