2010-10-20 19:08:46 +00:00
|
|
|
import logging
|
2010-10-17 20:04:50 +00:00
|
|
|
import msgpack
|
2010-10-20 19:08:46 +00:00
|
|
|
import os
|
2010-10-21 19:21:43 +00:00
|
|
|
|
2010-10-25 17:51:47 +00:00
|
|
|
from . import NS_ARCHIVE_CHUNKS, NS_CHUNK
|
2010-03-06 17:25:35 +00:00
|
|
|
|
|
|
|
|
|
|
|
class Cache(object):
|
|
|
|
"""Client Side cache
|
|
|
|
"""
|
|
|
|
|
2010-10-23 17:41:31 +00:00
|
|
|
def __init__(self, store, crypto):
|
2010-03-06 17:25:35 +00:00
|
|
|
self.store = store
|
2010-10-13 20:07:55 +00:00
|
|
|
self.path = os.path.join(os.path.expanduser('~'), '.dedupestore', 'cache',
|
2010-03-06 17:25:35 +00:00
|
|
|
'%s.cache' % self.store.uuid)
|
|
|
|
self.tid = -1
|
|
|
|
self.open()
|
|
|
|
if self.tid != self.store.tid:
|
2010-10-23 17:41:31 +00:00
|
|
|
self.init(crypto)
|
2010-03-06 17:25:35 +00:00
|
|
|
|
|
|
|
def open(self):
|
|
|
|
if not os.path.exists(self.path):
|
|
|
|
return
|
2010-10-21 22:10:51 +00:00
|
|
|
cache = msgpack.unpackb(open(self.path, 'rb').read())
|
2010-10-21 19:30:10 +00:00
|
|
|
version = cache.get('version')
|
2010-10-20 19:08:46 +00:00
|
|
|
if version != 1:
|
|
|
|
logging.error('Unsupported cache version %r' % version)
|
|
|
|
return
|
2010-10-21 19:30:10 +00:00
|
|
|
if cache['store'] != self.store.uuid:
|
2010-10-15 20:18:22 +00:00
|
|
|
raise Exception('Cache UUID mismatch')
|
2010-10-21 19:30:10 +00:00
|
|
|
self.chunkmap = cache['chunkmap']
|
|
|
|
self.tid = cache['tid']
|
2010-03-06 17:25:35 +00:00
|
|
|
|
2010-10-23 17:41:31 +00:00
|
|
|
def init(self, crypto):
|
2010-03-06 17:25:35 +00:00
|
|
|
"""Initializes cache by fetching and reading all archive indicies
|
|
|
|
"""
|
2010-10-21 19:21:43 +00:00
|
|
|
logging.info('Initializing cache...')
|
2010-03-06 17:25:35 +00:00
|
|
|
self.chunkmap = {}
|
|
|
|
self.tid = self.store.tid
|
|
|
|
if self.store.tid == 0:
|
|
|
|
return
|
2010-10-25 17:51:47 +00:00
|
|
|
for id in list(self.store.list(NS_ARCHIVE_CHUNKS)):
|
|
|
|
data, hash = crypto.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id))
|
2010-10-24 18:18:18 +00:00
|
|
|
cindex = msgpack.unpackb(data)
|
2010-10-21 19:21:43 +00:00
|
|
|
for id, size in cindex['chunks']:
|
2010-10-20 18:28:29 +00:00
|
|
|
try:
|
|
|
|
count, size = self.chunkmap[id]
|
|
|
|
self.chunkmap[id] = count + 1, size
|
|
|
|
except KeyError:
|
|
|
|
self.chunkmap[id] = 1, size
|
2010-10-17 20:04:50 +00:00
|
|
|
self.save()
|
|
|
|
|
2010-03-06 17:25:35 +00:00
|
|
|
def save(self):
|
2010-05-10 20:35:46 +00:00
|
|
|
assert self.store.state == self.store.OPEN
|
2010-10-21 19:30:10 +00:00
|
|
|
cache = {'version': 1,
|
2010-10-20 19:08:46 +00:00
|
|
|
'store': self.store.uuid,
|
2010-10-13 20:07:55 +00:00
|
|
|
'chunkmap': self.chunkmap,
|
2010-10-21 19:21:43 +00:00
|
|
|
'tid': self.store.tid,
|
|
|
|
}
|
2010-10-21 22:10:51 +00:00
|
|
|
data = msgpack.packb(cache)
|
2010-03-06 17:25:35 +00:00
|
|
|
cachedir = os.path.dirname(self.path)
|
|
|
|
if not os.path.exists(cachedir):
|
|
|
|
os.makedirs(cachedir)
|
|
|
|
with open(self.path, 'wb') as fd:
|
2010-10-21 19:30:10 +00:00
|
|
|
fd.write(data)
|
2010-03-06 17:25:35 +00:00
|
|
|
|
2010-10-23 19:38:42 +00:00
|
|
|
def add_chunk(self, id, data, crypto):
|
2010-03-15 20:23:34 +00:00
|
|
|
if self.seen_chunk(id):
|
|
|
|
return self.chunk_incref(id)
|
2010-10-24 18:18:18 +00:00
|
|
|
data, hash = crypto.encrypt_read(data)
|
2010-03-15 20:23:34 +00:00
|
|
|
csize = len(data)
|
2010-10-25 17:51:47 +00:00
|
|
|
self.store.put(NS_CHUNK, id, data)
|
2010-10-20 19:08:46 +00:00
|
|
|
self.chunkmap[id] = (1, csize)
|
|
|
|
return csize
|
2010-03-06 17:25:35 +00:00
|
|
|
|
2010-03-15 20:23:34 +00:00
|
|
|
def seen_chunk(self, id):
|
2010-10-17 15:44:41 +00:00
|
|
|
count, size = self.chunkmap.get(id, (0, 0))
|
2010-04-18 20:34:21 +00:00
|
|
|
return count
|
2010-03-06 17:25:35 +00:00
|
|
|
|
|
|
|
def chunk_incref(self, id):
|
2010-10-17 15:44:41 +00:00
|
|
|
count, size = self.chunkmap[id]
|
|
|
|
self.chunkmap[id] = (count + 1, size)
|
2010-10-20 18:28:29 +00:00
|
|
|
return size
|
2010-03-06 17:25:35 +00:00
|
|
|
|
|
|
|
def chunk_decref(self, id):
|
2010-10-17 15:44:41 +00:00
|
|
|
count, size = self.chunkmap[id]
|
2010-03-15 20:23:34 +00:00
|
|
|
if count == 1:
|
2010-03-06 17:25:35 +00:00
|
|
|
del self.chunkmap[id]
|
2010-10-25 17:51:47 +00:00
|
|
|
self.store.delete(NS_CHUNK, id)
|
2010-03-15 20:23:34 +00:00
|
|
|
else:
|
2010-10-17 15:44:41 +00:00
|
|
|
self.chunkmap[id] = (count - 1, size)
|
2010-10-13 20:07:55 +00:00
|
|
|
|
|
|
|
|