borg/dedupestore/cache.py

110 lines
3.4 KiB
Python

import hashlib
import os
import zlib
import msgpack
NS_ARCHIVES = 'ARCHIVES'
NS_CHUNKS = 'CHUNKS'
class Cache(object):
"""Client Side cache
"""
def __init__(self, store):
self.store = store
self.path = os.path.join(os.path.expanduser('~'), '.dedupestore', 'cache',
'%s.cache' % self.store.uuid)
self.tid = -1
self.open()
if self.tid != self.store.tid:
self.init()
def open(self):
if not os.path.exists(self.path):
return
data = open(self.path, 'rb').read()
id = data[:32]
data = data[32:]
if hashlib.sha256(data).digest() != id:
raise Exception('Cache hash did not match')
data = msgpack.unpackb(zlib.decompress(data))
if data['uuid'] != self.store.uuid:
raise Exception('Cache UUID mismatch')
self.chunkmap = data['chunkmap']
self.archives = data['archives']
self.tid = data['tid']
def init(self):
"""Initializes cache by fetching and reading all archive indicies
"""
self.chunkmap = {}
self.archives = {}
self.tid = self.store.tid
if self.store.tid == 0:
return
for id in list(self.store.list(NS_ARCHIVES)):
data = self.store.get(NS_ARCHIVES, id)
if hashlib.sha256(data).digest() != id:
raise Exception('Archive hash did not match')
archive = msgpack.unpackb(zlib.decompress(data))
self.archives[archive['name']] = id
for item in archive['items']:
if item['type'] != 'FILE':
continue
for idx in item['chunks']:
id, size = archive['chunks'][idx]
if self.seen_chunk(id):
self.chunk_incref(id)
else:
self.init_chunk(id, size)
self.save()
def save(self):
assert self.store.state == self.store.OPEN
data = {'uuid': self.store.uuid,
'chunkmap': self.chunkmap,
'tid': self.store.tid, 'archives': self.archives}
cachedir = os.path.dirname(self.path)
if not os.path.exists(cachedir):
os.makedirs(cachedir)
with open(self.path, 'wb') as fd:
data = zlib.compress(msgpack.packb(data))
id = hashlib.sha256(data).digest()
fd.write(id + data)
def add_chunk(self, data):
id = hashlib.sha256(data).digest()
if self.seen_chunk(id):
return self.chunk_incref(id)
osize = len(data)
data = zlib.compress(data)
data = hashlib.sha256(data).digest() + data
csize = len(data)
self.store.put(NS_CHUNKS, id, data)
return self.init_chunk(id, csize)
def init_chunk(self, id, size):
self.chunkmap[id] = (1, size)
return id, size
def seen_chunk(self, id):
count, size = self.chunkmap.get(id, (0, 0))
return count
def chunk_incref(self, id):
count, size = self.chunkmap[id]
self.chunkmap[id] = (count + 1, size)
return id, size
def chunk_decref(self, id):
count, size = self.chunkmap[id]
if count == 1:
del self.chunkmap[id]
self.store.delete(NS_CHUNKS, id)
else:
self.chunkmap[id] = (count - 1, size)