From 98f738fe7f1f101ee82cfc25794d0c97fd1157c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Tue, 9 Mar 2010 22:27:37 +0100 Subject: [PATCH] Some code refactoring. --- dedupestore/_speedups.c | 5 +- dedupestore/archiver.py | 158 +++++++++++++++++++++----------------- dedupestore/chunkifier.py | 2 +- 3 files changed, 91 insertions(+), 74 deletions(-) diff --git a/dedupestore/_speedups.c b/dedupestore/_speedups.c index 68b102084..62ca2da58 100644 --- a/dedupestore/_speedups.c +++ b/dedupestore/_speedups.c @@ -46,7 +46,7 @@ ChunkifyIter_dealloc(PyObject *self) { ChunkifyIter *c = (ChunkifyIter *)self; Py_DECREF(c->fd); - Py_DECREF(c->chunks); + Py_XDECREF(c->chunks); free(c->data); self->ob_type->tp_free(self); } @@ -65,6 +65,7 @@ ChunkifyIter_iternext(PyObject *self) if(c->extra) { c->done = 1; + Py_INCREF(c->extra); return c->extra; } for(;;) @@ -259,4 +260,4 @@ init_speedups(void) Py_INCREF(&ChunkifyIterType); PyModule_AddObject(m, "_ChunkifyIter", (PyObject *)&ChunkifyIterType); -} \ No newline at end of file +} diff --git a/dedupestore/archiver.py b/dedupestore/archiver.py index e7e39f4de..c2e6c39f9 100644 --- a/dedupestore/archiver.py +++ b/dedupestore/archiver.py @@ -10,79 +10,29 @@ from cache import Cache from store import Store, NS_ARCHIVES, NS_CHUNKS, CHUNK_SIZE -class Archiver(object): +class Archive(object): - def create_archive(self, archive_name, paths): - try: - self.store.get(NS_ARCHIVES, archive_name) - except Store.DoesNotExist: - pass - else: - raise Exception('Archive "%s" already exists' % archive_name) - items = [] - for path in paths: - for root, dirs, files in os.walk(path): - for d in dirs: - name = os.path.join(root, d) - items.append(self.process_dir(name, self.cache)) - for f in files: - name = os.path.join(root, f) - items.append(self.process_file(name, self.cache)) - archive = {'name': archive_name, 'items': items} - self.store.put(NS_ARCHIVES, archive_name, zlib.compress(cPickle.dumps(archive))) + def __init__(self, store, name=None): + self.store = store + self.items = [] + if name: + self.open(name) + + def open(self, name): + archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, name))) + self.items = archive['items'] + + def save(self, name): + archive = {'name': name, 'items': self.items} + self.store.put(NS_ARCHIVES, name, zlib.compress(cPickle.dumps(archive))) self.store.commit() - self.cache.archives.append(archive_name) - self.cache.save() - def delete_archive(self, archive_name): - try: - archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name))) - except Store.DoesNotExist: - raise Exception('Archive "%s" does not exist' % archive_name) - self.store.delete(NS_ARCHIVES, archive_name) - for item in archive['items']: - if item['type'] == 'FILE': - for c in item['chunks']: - self.cache.chunk_decref(c) - self.store.commit() - self.cache.archives.remove(archive_name) - self.cache.save() - - def list_archives(self): - print 'Archives:' - for archive in sorted(self.cache.archives): - print archive - - def list_archive(self, archive_name): - try: - archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name))) - except Store.DoesNotExist: - raise Exception('Archive "%s" does not exist' % archive_name) - for item in archive['items']: + def list(self): + for item in self.items: print item['path'] - def verify_archive(self, archive_name): - try: - archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name))) - except Store.DoesNotExist: - raise Exception('Archive "%s" does not exist' % archive_name) - for item in archive['items']: - if item['type'] == 'FILE': - print item['path'], '...', - for chunk in item['chunks']: - data = self.store.get(NS_CHUNKS, chunk) - if hashlib.sha1(data).digest() != chunk[4:]: - print 'ERROR' - break - else: - print 'OK' - - def extract_archive(self, archive_name): - try: - archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name))) - except Store.DoesNotExist: - raise Exception('Archive "%s" does not exist' % archive_name) - for item in archive['items']: + def extract(self): + for item in self.items: assert item['path'][0] not in ('/', '\\', ':') print item['path'] if item['type'] == 'DIR': @@ -96,15 +46,82 @@ class Archiver(object): raise Exception('Invalid chunk checksum') fd.write(zlib.decompress(data)) + def verify(self): + for item in self.items: + if item['type'] == 'FILE': + print item['path'], '...', + for chunk in item['chunks']: + data = self.store.get(NS_CHUNKS, chunk) + if hashlib.sha1(data).digest() != chunk[4:]: + print 'ERROR' + break + else: + print 'OK' + + def delete(self, cache): + self.store.delete(NS_ARCHIVES, self.name) + for item in self.items: + if item['type'] == 'FILE': + for c in item['chunks']: + cache.chunk_decref(c) + self.store.commit() + cache.archives.remove(self.name) + cache.save() + + +class Archiver(object): + + def create_archive(self, archive_name, paths): + try: + self.store.get(NS_ARCHIVES, archive_name) + except Store.DoesNotExist: + pass + else: + raise Exception('Archive "%s" already exists' % archive_name) + archive = Archive(self.store) + for path in paths: + for root, dirs, files in os.walk(path): + for d in dirs: + name = os.path.join(root, d) + archive.items.append(self.process_dir(name, self.cache)) + for f in files: + name = os.path.join(root, f) + archive.items.append(self.process_file(name, self.cache)) + archive.save(archive_name) + self.cache.archives.append(archive_name) + self.cache.save() + + def delete_archive(self, archive_name): + archive = Archive(self.store, archive_name) + archive.delete(self.cache) + + def list_archives(self): + print 'Archives:' + for archive in sorted(self.cache.archives): + print archive + + def list_archive(self, archive_name): + archive = Archive(self.store, archive_name) + archive.list() + + def verify_archive(self, archive_name): + archive = Archive(self.store, archive_name) + archive.verify() + + def extract_archive(self, archive_name): + archive = Archive(self.store, archive_name) + archive.extract() + def process_dir(self, path, cache): path = path.lstrip('/\\:') print 'Directory: %s' % (path) return {'type': 'DIR', 'path': path} def process_file(self, path, cache): - print 'Adding: %s...' % path, - sys.stdout.flush() with open(path, 'rb') as fd: + path = path.lstrip('/\\:') + print 'Adding: %s...' % path, + sys.stdout.flush() origsize = 0 compsize = 0 chunks = [] @@ -113,7 +130,6 @@ class Archiver(object): id, size = cache.add_chunk(chunk) compsize += size chunks.append(id) - path = path.lstrip('/\\:') ratio = origsize and compsize * 100 / origsize or 0 print '(%d chunks: %d%%)' % (len(chunks), ratio) return {'type': 'FILE', 'path': path, 'size': origsize, 'chunks': chunks} diff --git a/dedupestore/chunkifier.py b/dedupestore/chunkifier.py index 10222832d..c1e693b68 100644 --- a/dedupestore/chunkifier.py +++ b/dedupestore/chunkifier.py @@ -29,6 +29,7 @@ def roll_checksum(sum, remove, add, len): class ChunkifyIter(object): + def __init__(self, fd, chunk_size, chunks): self.fd = fd self.chunk_size = chunk_size @@ -140,6 +141,5 @@ except ImportError: if __name__ == '__main__': - import StringIO import doctest doctest.testmod()