borg/darc/cache.py

182 lines
6.6 KiB
Python
Raw Normal View History

2010-12-21 20:29:09 +00:00
from ConfigParser import RawConfigParser
import fcntl
import msgpack
2010-10-20 19:08:46 +00:00
import os
2010-12-21 20:29:09 +00:00
import shutil
2010-12-06 20:46:29 +00:00
from . import NS_ARCHIVE_CHUNKS, NS_CHUNK, PACKET_ARCHIVE_CHUNKS, PACKET_CHUNK
2010-12-21 20:29:09 +00:00
from .hashindex import NSIndex
2010-03-06 17:25:35 +00:00
class Cache(object):
"""Client Side cache
"""
def __init__(self, store, keychain):
2010-12-21 20:29:09 +00:00
self.txn_active = False
2010-03-06 17:25:35 +00:00
self.store = store
self.keychain = keychain
2010-12-21 20:29:09 +00:00
self.path = os.path.join(Cache.cache_dir_path(), self.store.id.encode('hex'))
if not os.path.exists(self.path):
self.create()
2010-03-06 17:25:35 +00:00
self.open()
2010-12-21 20:29:09 +00:00
assert self.id == store.id
if self.tid != store.tid:
2010-12-21 20:29:09 +00:00
self.sync()
2010-03-06 17:25:35 +00:00
@staticmethod
def cache_dir_path():
"""Return path to directory used for storing users cache files"""
return os.path.join(os.path.expanduser('~'), '.darc', 'cache')
2010-12-21 20:29:09 +00:00
def create(self):
"""Create a new empty store at `path`
"""
2011-01-04 22:00:39 +00:00
os.makedirs(self.path)
2010-12-21 20:29:09 +00:00
with open(os.path.join(self.path, 'README'), 'wb') as fd:
fd.write('This is a DARC cache')
config = RawConfigParser()
config.add_section('cache')
config.set('cache', 'version', '1')
config.set('cache', 'store_id', self.store.id.encode('hex'))
config.set('cache', 'tid', '0')
with open(os.path.join(self.path, 'config'), 'wb') as fd:
config.write(fd)
NSIndex.create(os.path.join(self.path, 'chunks'))
with open(os.path.join(self.path, 'files'), 'wb') as fd:
pass # empty file
2010-03-06 17:25:35 +00:00
def open(self):
2010-12-21 20:29:09 +00:00
if not os.path.isdir(self.path):
raise Exception('%s Does not look like a darc cache' % self.path)
self.lock_fd = open(os.path.join(self.path, 'README'), 'r+')
fcntl.flock(self.lock_fd, fcntl.LOCK_EX)
self.rollback()
self.config = RawConfigParser()
self.config.read(os.path.join(self.path, 'config'))
if self.config.getint('cache', 'version') != 1:
raise Exception('%s Does not look like a darc cache')
self.id = self.config.get('cache', 'store_id').decode('hex')
self.tid = self.config.getint('cache', 'tid')
self.chunks = NSIndex(os.path.join(self.path, 'chunks'))
with open(os.path.join(self.path, 'files'), 'rb') as fd:
self.files = {}
u = msgpack.Unpacker()
while True:
data = fd.read(64 * 1024)
if not data:
break
u.feed(data)
for hash, item in u:
if item[0] < 8:
self.files[hash] = (item[0] + 1,) + item[1:]
def begin_txn(self):
# Initialize transaction snapshot
txn_dir = os.path.join(self.path, 'txn.tmp')
os.mkdir(txn_dir)
shutil.copy(os.path.join(self.path, 'config'), txn_dir)
shutil.copy(os.path.join(self.path, 'chunks'), txn_dir)
shutil.copy(os.path.join(self.path, 'files'), txn_dir)
os.rename(os.path.join(self.path, 'txn.tmp'),
os.path.join(self.path, 'txn.active'))
self.txn_active = True
def commit(self):
"""Commit transaction
"""
2011-01-04 22:16:55 +00:00
if not self.txn_active:
return
2010-12-21 20:29:09 +00:00
with open(os.path.join(self.path, 'files'), 'wb') as fd:
for item in self.files.iteritems():
msgpack.pack(item, fd)
for id, (count, size) in self.chunks.iteritems():
if count > 1000000:
self.chunks[id] = count - 1000000, size
self.config.set('cache', 'tid', self.store.tid)
with open(os.path.join(self.path, 'config'), 'w') as fd:
self.config.write(fd)
self.chunks.flush()
os.rename(os.path.join(self.path, 'txn.active'),
os.path.join(self.path, 'txn.tmp'))
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
self.txn_active = False
def rollback(self):
"""Roll back partial and aborted transactions
"""
# Remove partial transaction
if os.path.exists(os.path.join(self.path, 'txn.tmp')):
shutil.rmtree(os.path.join(self.path, 'txn.tmp'))
# Roll back active transaction
txn_dir = os.path.join(self.path, 'txn.active')
if os.path.exists(txn_dir):
shutil.copy(os.path.join(txn_dir, 'config'), self.path)
shutil.copy(os.path.join(txn_dir, 'chunks'), self.path)
shutil.copy(os.path.join(txn_dir, 'files'), self.path)
shutil.rmtree(txn_dir)
self.txn_active = False
def sync(self):
2010-03-06 17:25:35 +00:00
"""Initializes cache by fetching and reading all archive indicies
"""
2010-12-21 20:29:09 +00:00
self.begin_txn()
2010-10-30 11:44:25 +00:00
print 'Initializing cache...'
for id in self.store.list(NS_ARCHIVE_CHUNKS):
2010-12-06 20:46:29 +00:00
magic, data, hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id))
assert magic == PACKET_ARCHIVE_CHUNKS
2010-12-05 16:57:17 +00:00
chunks = msgpack.unpackb(data)
for id, size in chunks:
2010-10-20 18:28:29 +00:00
try:
2010-12-21 20:29:09 +00:00
count, size = self.chunks[id]
self.chunks[id] = count + 1, size
2010-10-20 18:28:29 +00:00
except KeyError:
2010-12-21 20:29:09 +00:00
self.chunks[id] = 1, size
2010-03-06 17:25:35 +00:00
2010-10-27 17:30:21 +00:00
def add_chunk(self, id, data):
2010-12-21 20:29:09 +00:00
if not self.txn_active:
self.begin_txn()
if self.seen_chunk(id):
return self.chunk_incref(id)
2010-12-06 20:46:29 +00:00
data, hash = self.keychain.encrypt(PACKET_CHUNK, data)
csize = len(data)
self.store.put(NS_CHUNK, id, data)
2010-12-21 20:29:09 +00:00
self.chunks[id] = (1000001, csize)
2010-11-23 13:46:53 +00:00
return id
2010-03-06 17:25:35 +00:00
def seen_chunk(self, id):
2010-12-21 20:29:09 +00:00
return self.chunks.get(id, (0, 0))[0]
2010-03-06 17:25:35 +00:00
def chunk_incref(self, id):
2010-12-21 20:29:09 +00:00
if not self.txn_active:
self.begin_txn()
count, size = self.chunks[id]
2010-11-23 13:46:53 +00:00
if count < 1000000:
2010-12-21 20:29:09 +00:00
self.chunks[id] = (count + 1000001, size)
2010-11-23 13:46:53 +00:00
return id
2010-03-06 17:25:35 +00:00
def chunk_decref(self, id):
2010-12-21 20:29:09 +00:00
if not self.txn_active:
self.begin_txn()
count, size = self.chunks[id]
if count == 1:
2010-12-21 20:29:09 +00:00
del self.chunks[id]
self.store.delete(NS_CHUNK, id)
else:
2010-12-21 20:29:09 +00:00
self.chunks[id] = (count - 1, size)
def file_known_and_unchanged(self, path_hash, st):
2010-12-21 20:29:09 +00:00
entry = self.files.get(path_hash)
2010-10-26 18:50:30 +00:00
if (entry and entry[3] == st.st_mtime
and entry[2] == st.st_size and entry[1] == st.st_ino):
# reset entry age
2010-12-21 20:29:09 +00:00
self.files[path_hash] = (0,) + entry[1:]
return entry[4], entry[2]
else:
return None, 0
2010-12-21 20:29:09 +00:00
def memorize_file(self, path_hash, st, ids):
# Entry: Age, inode, size, mtime, chunk ids
2010-12-21 20:29:09 +00:00
self.files[path_hash] = 0, st.st_ino, st.st_size, st.st_mtime, ids