mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-01-03 05:35:58 +00:00
Code Issues Releases Wiki Activity

bandstore: More robust band cleaning

Browse source
This commit is contained in:
Jonas Borgström 2010-10-19 00:00:52 +02:00
parent be649e8e52
commit 52870eb2d8
2 changed files with 21 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
dedupestore/archiver.py
View file

@ -98,7 +98,7 @@ def extract(self, dest=None):
dest = dest or os.getcwdu() dest = dest or os.getcwdu()
for item in self.items: for item in self.items:
assert item['path'][0] not in ('/', '\\', ':') assert item['path'][0] not in ('/', '\\', ':')
path = os.path.join(dest, item['path']) path = os.path.join(dest, item['path'].decode('utf-8'))
if item['type'] == 'DIRECTORY': if item['type'] == 'DIRECTORY':
logging.info(path) logging.info(path)
if not os.path.exists(path): if not os.path.exists(path):
@ -127,6 +127,7 @@ def extract(self, dest=None):
def verify(self): def verify(self):
for item in self.items: for item in self.items:
if item['type'] == 'FILE': if item['type'] == 'FILE':
item['path'] = item['path'].decode('utf-8')
for chunk in item['chunks']: for chunk in item['chunks']:
id = self.chunk_idx[chunk] id = self.chunk_idx[chunk]
data = self.store.get(NS_CHUNKS, id) data = self.store.get(NS_CHUNKS, id)

26
dedupestore/bandstore.py
View file

@ -62,6 +62,7 @@ def _begin(self):
while os.path.exists(self.band_filename(band)): while os.path.exists(self.band_filename(band)):
os.unlink(self.band_filename(band)) os.unlink(self.band_filename(band))
band += 1 band += 1
self.delete_bands()
def create(self, path): def create(self, path):
os.mkdir(path) os.mkdir(path)
@ -69,11 +70,14 @@ def create(self, path):
cnx = sqlite3.connect(os.path.join(path, 'dedupestore.db')) cnx = sqlite3.connect(os.path.join(path, 'dedupestore.db'))
cnx.execute('CREATE TABLE objects(ns TEXT NOT NULL, id NOT NULL, ' cnx.execute('CREATE TABLE objects(ns TEXT NOT NULL, id NOT NULL, '
'band NOT NULL, offset NOT NULL, size NOT NULL)') 'band NOT NULL, offset NOT NULL, size NOT NULL)')
cnx.execute('CREATE UNIQUE INDEX objects_pk ON objects(ns, id)')
cnx.execute('CREATE TABLE to_delete(band NOT NULL)')
cnx.execute('CREATE UNIQUE INDEX to_delete_pk ON to_delete(band)')
cnx.execute('CREATE TABLE system(uuid NOT NULL, tid NOT NULL, ' cnx.execute('CREATE TABLE system(uuid NOT NULL, tid NOT NULL, '
'nextband NOT NULL, version NOT NULL, bandlimit NOT NULL)') 'nextband NOT NULL, version NOT NULL, bandlimit NOT NULL)')
cnx.execute('INSERT INTO system VALUES(?,?,?,?,?)', cnx.execute('INSERT INTO system VALUES(?,?,?,?,?)',
(uuid.uuid1().hex, 0, 0, 1, self.BAND_LIMIT)) (uuid.uuid1().hex, 0, 0, 1, self.BAND_LIMIT))
cnx.execute('CREATE UNIQUE INDEX objects_pk ON objects(ns, id)') cnx.commit()
def close(self): def close(self):
self.rollback() self.rollback()
@ -85,7 +89,18 @@ def commit(self):
""" """
""" """
self.band = None self.band = None
for b in self.to_delete: self.cursor.executemany('INSERT INTO to_delete(band) VALUES(?)',
[[d] for d in self.to_delete])
self.cursor.execute('UPDATE system SET tid=tid+1, nextband=?',
(self.nextband,))
self.cnx.commit()
self.tid += 1
self._begin()
def delete_bands(self):
self.cursor.execute('SELECT band FROM to_delete')
to_delete = [r[0] for r in self.cursor.fetchall()]
for b in to_delete:
objects = self.cursor.execute('SELECT ns, id, offset, size ' objects = self.cursor.execute('SELECT ns, id, offset, size '
'FROM objects WHERE band=? ORDER BY offset', 'FROM objects WHERE band=? ORDER BY offset',
(b,)).fetchall() (b,)).fetchall()
@ -93,13 +108,10 @@ def commit(self):
band, offset, size = self.store_data(self.retrieve_data(b, *o[2:])) band, offset, size = self.store_data(self.retrieve_data(b, *o[2:]))
self.cursor.execute('UPDATE objects SET band=?, offset=?, size=? ' self.cursor.execute('UPDATE objects SET band=?, offset=?, size=? '
'WHERE ns=? AND id=?', (band, offset, size, o[0], o[1])) 'WHERE ns=? AND id=?', (band, offset, size, o[0], o[1]))
self.cursor.execute('DELETE FROM to_delete WHERE band=?', (b,))
self.cursor.execute('UPDATE system SET nextband=?', (self.nextband,))
self.cnx.commit() self.cnx.commit()
os.unlink(self.band_filename(b)) os.unlink(self.band_filename(b))
self.cursor.execute('UPDATE system SET tid=tid+1, nextband=?',
(self.nextband,))
self.cnx.commit()
self.tid += 1
self._begin()
def rollback(self): def rollback(self):
""" """