diff --git a/darc/__init__.py b/darc/__init__.py index 378bd30f2..9d705b9bd 100644 --- a/darc/__init__.py +++ b/darc/__init__.py @@ -5,3 +5,10 @@ NS_ARCHIVE_METADATA = 1 NS_ARCHIVE_CHUNKS = 2 NS_ARCHIVE_ITEMS = 3 +PACKET_ENCRYPT_READ = 2 ** 7 +PACKET_ENCRYPT_CREATE = 2 ** 6 +PACKET_CHUNK = 1 | PACKET_ENCRYPT_READ +PACKET_ARCHIVE_METADATA = 2 | PACKET_ENCRYPT_READ +PACKET_ARCHIVE_ITEMS = 3 | PACKET_ENCRYPT_READ +PACKET_ARCHIVE_CHUNKS = 1 | PACKET_ENCRYPT_CREATE + diff --git a/darc/archive.py b/darc/archive.py index b4520470a..30335cc16 100644 --- a/darc/archive.py +++ b/darc/archive.py @@ -7,7 +7,8 @@ import stat import sys from xattr import xattr, XATTR_NOFOLLOW -from . import NS_ARCHIVE_METADATA, NS_ARCHIVE_ITEMS, NS_ARCHIVE_CHUNKS, NS_CHUNK +from . import NS_ARCHIVE_METADATA, NS_ARCHIVE_ITEMS, NS_ARCHIVE_CHUNKS, NS_CHUNK, \ + PACKET_ARCHIVE_METADATA, PACKET_ARCHIVE_ITEMS, PACKET_ARCHIVE_CHUNKS, PACKET_CHUNK from ._speedups import chunkify from .helpers import uid2user, user2uid, gid2group, group2gid, IntegrityError @@ -35,15 +36,17 @@ class Archive(object): def load(self, id): self.id = id try: - data, self.hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_METADATA, self.id)) + kind, data, self.hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_METADATA, self.id)) except self.store.DoesNotExist: raise self.DoesNotExist + assert kind == PACKET_ARCHIVE_METADATA self.metadata = msgpack.unpackb(data) assert self.metadata['version'] == 1 def get_chunks(self): for id in self.metadata['chunks_ids']: - data, hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id)) + magic, data, hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id)) + assert magic == PACKET_ARCHIVE_CHUNKS assert hash == id chunks = msgpack.unpackb(data) for chunk in chunks: @@ -51,7 +54,8 @@ class Archive(object): def get_items(self): for id in self.metadata['items_ids']: - data, items_hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_ITEMS, id)) + magic, data, items_hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_ITEMS, id)) + assert magic == PACKET_ARCHIVE_ITEMS assert items_hash == id items = msgpack.unpackb(data) for item in items: @@ -63,7 +67,7 @@ class Archive(object): self.flush_items() def flush_items(self): - data, hash = self.keychain.encrypt_read(msgpack.packb(self.items)) + data, hash = self.keychain.encrypt(PACKET_ARCHIVE_ITEMS, msgpack.packb(self.items)) self.store.put(NS_ARCHIVE_ITEMS, hash, data) self.items_ids.append(hash) self.items = [] @@ -72,7 +76,7 @@ class Archive(object): chunks = [] ids = [] def flush(chunks): - data, hash = self.keychain.encrypt_create(msgpack.packb(chunks)) + data, hash = self.keychain.encrypt(PACKET_ARCHIVE_CHUNKS, msgpack.packb(chunks)) self.store.put(NS_ARCHIVE_CHUNKS, hash, data) ids.append(hash) for id, (count, size) in cache.chunk_counts.iteritems(): @@ -98,7 +102,7 @@ class Archive(object): 'username': getuser(), 'time': datetime.utcnow().isoformat(), } - data, self.hash = self.keychain.encrypt_read(msgpack.packb(metadata)) + data, self.hash = self.keychain.encrypt(PACKET_ARCHIVE_METADATA, msgpack.packb(metadata)) self.store.put(NS_ARCHIVE_METADATA, self.id, data) self.store.commit() @@ -149,7 +153,8 @@ class Archive(object): with open(path, 'wb') as fd: for id in item['chunks']: try: - data, hash = self.keychain.decrypt(self.store.get(NS_CHUNK, id)) + magic, data, hash = self.keychain.decrypt(self.store.get(NS_CHUNK, id)) + assert magic == PACKET_CHUNK if self.keychain.id_hash(data) != id: raise IntegrityError('chunk hash did not match') fd.write(data) @@ -185,7 +190,8 @@ class Archive(object): def verify_file(self, item): for id in item['chunks']: try: - data, hash = self.keychain.decrypt(self.store.get(NS_CHUNK, id)) + magic, data, hash = self.keychain.decrypt(self.store.get(NS_CHUNK, id)) + assert magic == PACKET_CHUNK if self.keychain.id_hash(data) != id: raise IntegrityError('chunk id did not match') except IntegrityError: diff --git a/darc/cache.py b/darc/cache.py index 4fe23df51..0a4f01725 100644 --- a/darc/cache.py +++ b/darc/cache.py @@ -1,7 +1,7 @@ import msgpack import os -from . import NS_ARCHIVE_CHUNKS, NS_CHUNK +from . import NS_ARCHIVE_CHUNKS, NS_CHUNK, PACKET_ARCHIVE_CHUNKS, PACKET_CHUNK class Cache(object): @@ -27,8 +27,8 @@ class Cache(object): if not os.path.exists(self.path): return with open(self.path, 'rb') as fd: - data, hash = self.keychain.decrypt(fd.read()) - cache = msgpack.unpackb(data) + #data, hash = self.keychain.decrypt(fd.read()) + cache = msgpack.unpackb(fd.read()) assert cache['version'] == 1 self.chunk_counts = cache['chunk_counts'] self.file_chunks = cache['file_chunks'] @@ -44,7 +44,8 @@ class Cache(object): if len(id) != 32: import ipdb ipdb.set_trace() - data, hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id)) + magic, data, hash = self.keychain.decrypt(self.store.get(NS_ARCHIVE_CHUNKS, id)) + assert magic == PACKET_ARCHIVE_CHUNKS chunks = msgpack.unpackb(data) for id, size in chunks: try: @@ -69,17 +70,17 @@ class Cache(object): 'chunk_counts': self.chunk_counts, 'file_chunks': dict(self.filter_file_chunks()), } - data, hash = self.keychain.encrypt_create(msgpack.packb(cache)) +# data, hash = self.keychain.encrypt_create(msgpack.packb(cache)) cachedir = os.path.dirname(self.path) if not os.path.exists(cachedir): os.makedirs(cachedir) with open(self.path, 'wb') as fd: - fd.write(data) + fd.write(msgpack.packb(cache)) def add_chunk(self, id, data): if self.seen_chunk(id): return self.chunk_incref(id) - data, hash = self.keychain.encrypt_read(data) + data, hash = self.keychain.encrypt(PACKET_CHUNK, data) csize = len(data) self.store.put(NS_CHUNK, id, data) self.chunk_counts[id] = (1000001, csize) diff --git a/darc/keychain.py b/darc/keychain.py index 9486ea2c7..896bd297f 100644 --- a/darc/keychain.py +++ b/darc/keychain.py @@ -11,6 +11,7 @@ from Crypto.PublicKey import RSA from Crypto.Util import Counter from Crypto.Util.number import bytes_to_long, long_to_bytes +from . import PACKET_ENCRYPT_READ, PACKET_ENCRYPT_CREATE from .helpers import IntegrityError, zero_pad from .oaep import OAEP @@ -140,28 +141,21 @@ class Keychain(object): """ return HMAC.new(self.aes_id, data, SHA256).digest() - def _encrypt(self, id, rsa_key, key, data): + def encrypt(self, magic, data): """Helper function used by `encrypt_read` and `encrypt_create` """ data = zlib.compress(data) nonce = long_to_bytes(self.counter.next_value(), 8) - data = nonce + rsa_key + AES.new(key, AES.MODE_CTR, '', counter=self.counter).encrypt(data) + if magic & PACKET_ENCRYPT_READ: + data = ''.join((nonce, self.read_encrypted, + AES.new(self.read_key, AES.MODE_CTR, '', + counter=self.counter).encrypt(data))) + elif magic & PACKET_ENCRYPT_CREATE: + data = ''.join((nonce, self.create_encrypted, + AES.new(self.create_key, AES.MODE_CTR, '', + counter=self.counter).encrypt(data))) hash = self.id_hash(data) - return ''.join((id, hash, data)), hash - - def encrypt_read(self, data): - """Encrypt `data` using the AES "read" key - - An RSA encrypted version of the AES key is included in the header - """ - return self._encrypt(self.READ, self.read_encrypted, self.read_key, data) - - def encrypt_create(self, data, iv=None): - """Encrypt `data` using the AES "create" key - - An RSA encrypted version of the AES key is included in the header - """ - return self._encrypt(self.CREATE, self.create_encrypted, self.create_key, data) + return ''.join((chr(magic), hash, data)), hash def _decrypt_key(self, data, rsa_key): """Helper function used by `decrypt` @@ -175,20 +169,20 @@ class Keychain(object): def decrypt(self, data): """Decrypt `data` previously encrypted by `encrypt_create` or `encrypt_read` """ - type = data[0] + magic = ord(data[0]) hash = data[1:33] if self.id_hash(data[33:]) != hash: raise IntegrityError('Encryption integrity error') nonce = bytes_to_long(data[33:41]) counter = Counter.new(64, prefix='\0' * 8, initial_value=nonce) - if type == self.READ: + if magic & PACKET_ENCRYPT_READ: key = self._decrypt_key(data[41:297], self.rsa_read) - elif type == self.CREATE: + elif magic & PACKET_ENCRYPT_CREATE: key = self._decrypt_key(data[41:297], self.rsa_create) else: - raise Exception('Unknown pack type %d found' % ord(type)) + raise Exception('Unknown pack magic %d found' % magic) data = AES.new(key, AES.MODE_CTR, counter=counter).decrypt(data[297:]) - return zlib.decompress(data), hash + return magic, zlib.decompress(data), hash diff --git a/darc/store.py b/darc/store.py index 50904f8c0..6f24d3d3d 100644 --- a/darc/store.py +++ b/darc/store.py @@ -66,7 +66,7 @@ class Store(object): self.config.read(os.path.join(path, 'config')) if self.config.getint('store', 'version') != 1: raise Exception('%s Does not look like a darc store') - self.id = self.config.get('store', 'id') + self.id = self.config.get('store', 'id').decode('hex') self.tid = self.config.getint('state', 'tid') next_band = self.config.getint('state', 'next_band') max_band_size = self.config.getint('store', 'max_band_size') @@ -287,7 +287,6 @@ class HashIndex(DictMixin): def resize(self, capacity=0): capacity = capacity or self.buckets.size * 2 - print 'resizing to', capacity if capacity < self.num_entries: raise ValueError('HashIndex full') new = HashIndex.create(self.path + '.tmp', capacity) @@ -341,6 +340,7 @@ class BandIO(object): fd.seek(offset) data = fd.read(self.header_fmt.size) size, magic, ns, id = self.header_fmt.unpack(data) + assert magic == 0 return fd.read(size - self.header_fmt.size) def iter_objects(self, band):