mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Some code refactoring.

This commit is contained in:
Jonas Borgström 2010-03-09 22:27:37 +01:00
parent 723c636f06
commit 98f738fe7f
3 changed files with 91 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dedupestore/_speedups.c
View File

@ -46,7 +46,7 @@ ChunkifyIter_dealloc(PyObject *self)
{
ChunkifyIter *c = (ChunkifyIter *)self;
Py_DECREF(c->fd);
Py_DECREF(c->chunks);
Py_XDECREF(c->chunks);
free(c->data);
self->ob_type->tp_free(self);
}
@ -65,6 +65,7 @@ ChunkifyIter_iternext(PyObject *self)
if(c->extra)
{
c->done = 1;
Py_INCREF(c->extra);
return c->extra;
}
for(;;)
@ -259,4 +260,4 @@ init_speedups(void)
Py_INCREF(&ChunkifyIterType);
PyModule_AddObject(m, "_ChunkifyIter", (PyObject *)&ChunkifyIterType);
}
}

158
dedupestore/archiver.py
View File

@ -10,79 +10,29 @@ from cache import Cache
from store import Store, NS_ARCHIVES, NS_CHUNKS, CHUNK_SIZE
class Archiver(object):
class Archive(object):
def create_archive(self, archive_name, paths):
try:
self.store.get(NS_ARCHIVES, archive_name)
except Store.DoesNotExist:
pass
else:
raise Exception('Archive "%s" already exists' % archive_name)
items = []
for path in paths:
for root, dirs, files in os.walk(path):
for d in dirs:
name = os.path.join(root, d)
items.append(self.process_dir(name, self.cache))
for f in files:
name = os.path.join(root, f)
items.append(self.process_file(name, self.cache))
archive = {'name': archive_name, 'items': items}
self.store.put(NS_ARCHIVES, archive_name, zlib.compress(cPickle.dumps(archive)))
def __init__(self, store, name=None):
self.store = store
self.items = []
if name:
self.open(name)
def open(self, name):
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, name)))
self.items = archive['items']
def save(self, name):
archive = {'name': name, 'items': self.items}
self.store.put(NS_ARCHIVES, name, zlib.compress(cPickle.dumps(archive)))
self.store.commit()
self.cache.archives.append(archive_name)
self.cache.save()
def delete_archive(self, archive_name):
try:
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
except Store.DoesNotExist:
raise Exception('Archive "%s" does not exist' % archive_name)
self.store.delete(NS_ARCHIVES, archive_name)
for item in archive['items']:
if item['type'] == 'FILE':
for c in item['chunks']:
self.cache.chunk_decref(c)
self.store.commit()
self.cache.archives.remove(archive_name)
self.cache.save()
def list_archives(self):
print 'Archives:'
for archive in sorted(self.cache.archives):
print archive
def list_archive(self, archive_name):
try:
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
except Store.DoesNotExist:
raise Exception('Archive "%s" does not exist' % archive_name)
for item in archive['items']:
def list(self):
for item in self.items:
print item['path']
def verify_archive(self, archive_name):
try:
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
except Store.DoesNotExist:
raise Exception('Archive "%s" does not exist' % archive_name)
for item in archive['items']:
if item['type'] == 'FILE':
print item['path'], '...',
for chunk in item['chunks']:
data = self.store.get(NS_CHUNKS, chunk)
if hashlib.sha1(data).digest() != chunk[4:]:
print 'ERROR'
break
else:
print 'OK'
def extract_archive(self, archive_name):
try:
archive = cPickle.loads(zlib.decompress(self.store.get(NS_ARCHIVES, archive_name)))
except Store.DoesNotExist:
raise Exception('Archive "%s" does not exist' % archive_name)
for item in archive['items']:
def extract(self):
for item in self.items:
assert item['path'][0] not in ('/', '\\', ':')
print item['path']
if item['type'] == 'DIR':
@ -96,15 +46,82 @@ class Archiver(object):
raise Exception('Invalid chunk checksum')
fd.write(zlib.decompress(data))
def verify(self):
for item in self.items:
if item['type'] == 'FILE':
print item['path'], '...',
for chunk in item['chunks']:
data = self.store.get(NS_CHUNKS, chunk)
if hashlib.sha1(data).digest() != chunk[4:]:
print 'ERROR'
break
else:
print 'OK'
def delete(self, cache):
self.store.delete(NS_ARCHIVES, self.name)
for item in self.items:
if item['type'] == 'FILE':
for c in item['chunks']:
cache.chunk_decref(c)
self.store.commit()
cache.archives.remove(self.name)
cache.save()
class Archiver(object):
def create_archive(self, archive_name, paths):
try:
self.store.get(NS_ARCHIVES, archive_name)
except Store.DoesNotExist:
pass
else:
raise Exception('Archive "%s" already exists' % archive_name)
archive = Archive(self.store)
for path in paths:
for root, dirs, files in os.walk(path):
for d in dirs:
name = os.path.join(root, d)
archive.items.append(self.process_dir(name, self.cache))
for f in files:
name = os.path.join(root, f)
archive.items.append(self.process_file(name, self.cache))
archive.save(archive_name)
self.cache.archives.append(archive_name)
self.cache.save()
def delete_archive(self, archive_name):
archive = Archive(self.store, archive_name)
archive.delete(self.cache)
def list_archives(self):
print 'Archives:'
for archive in sorted(self.cache.archives):
print archive
def list_archive(self, archive_name):
archive = Archive(self.store, archive_name)
archive.list()
def verify_archive(self, archive_name):
archive = Archive(self.store, archive_name)
archive.verify()
def extract_archive(self, archive_name):
archive = Archive(self.store, archive_name)
archive.extract()
def process_dir(self, path, cache):
path = path.lstrip('/\\:')
print 'Directory: %s' % (path)
return {'type': 'DIR', 'path': path}
def process_file(self, path, cache):
print 'Adding: %s...' % path,
sys.stdout.flush()
with open(path, 'rb') as fd:
path = path.lstrip('/\\:')
print 'Adding: %s...' % path,
sys.stdout.flush()
origsize = 0
compsize = 0
chunks = []
@ -113,7 +130,6 @@ class Archiver(object):
id, size = cache.add_chunk(chunk)
compsize += size
chunks.append(id)
path = path.lstrip('/\\:')
ratio = origsize and compsize * 100 / origsize or 0
print '(%d chunks: %d%%)' % (len(chunks), ratio)
return {'type': 'FILE', 'path': path, 'size': origsize, 'chunks': chunks}

2
dedupestore/chunkifier.py
View File

@ -29,6 +29,7 @@ def roll_checksum(sum, remove, add, len):
class ChunkifyIter(object):
def __init__(self, fd, chunk_size, chunks):
self.fd = fd
self.chunk_size = chunk_size
@ -140,6 +141,5 @@ except ImportError:
if __name__ == '__main__':
import StringIO
import doctest
doctest.testmod()