mirror/borg
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2025-01-03 05:35:58 +00:00
Code Issues Releases Wiki Activity

Various banded store improvements.

Browse source
This commit is contained in:
Jonas Borgström 2010-10-17 22:42:52 +02:00
parent a8ff4362b8
commit b99ea117e1
1 changed files with 29 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
dedupestore/bandstore.py
View file

@ -24,7 +24,8 @@ class AlreadyExists(KeyError):
BAND_LIMIT = 1 * 1024 * 1024 BAND_LIMIT = 1 * 1024 * 1024
def __init__(self, path): def __init__(self, path):
self.band_fd = None self.read_fd = None
self.write_fd = None
if not os.path.exists(path): if not os.path.exists(path):
self.create(path) self.create(path)
self.open(path) self.open(path)
@ -43,15 +44,19 @@ def open(self, path):
self._begin() self._begin()
def _begin(self): def _begin(self):
if self.band_fd: if self.read_fd:
self.band_fd.close() self.read_fd.close()
self.band_fd = None self.read_fd = None
if self.write_fd:
self.write_fd.close()
self.write_fd = None
row = self.cursor.execute('SELECT uuid, tid, nextband, version, ' row = self.cursor.execute('SELECT uuid, tid, nextband, version, '
'bandlimit FROM system').fetchone() 'bandlimit FROM system').fetchone()
self.uuid, self.tid, self.nextband, version, self.bandlimit = row self.uuid, self.tid, self.nextband, version, self.bandlimit = row
assert version == 1 assert version == 1
self.state = self.OPEN self.state = self.OPEN
self.band = None self.read_band = None
self.write_band = None
self.to_delete = set() self.to_delete = set()
band = self.nextband band = self.nextband
while os.path.exists(self.band_filename(band)): while os.path.exists(self.band_filename(band)):
@ -88,6 +93,7 @@ def commit(self):
band, offset, size = self.store_data(self.retrieve_data(b, *o[2:])) band, offset, size = self.store_data(self.retrieve_data(b, *o[2:]))
self.cursor.execute('UPDATE objects SET band=?, offset=?, size=? ' self.cursor.execute('UPDATE objects SET band=?, offset=?, size=? '
'WHERE ns=? AND id=?', (band, offset, size, o[0], o[1])) 'WHERE ns=? AND id=?', (band, offset, size, o[0], o[1]))
self.cnx.commit()
os.unlink(self.band_filename(b)) os.unlink(self.band_filename(b))
self.cursor.execute('UPDATE system SET tid=tid+1, nextband=?', self.cursor.execute('UPDATE system SET tid=tid+1, nextband=?',
(self.nextband,)) (self.nextband,))
@ -116,26 +122,28 @@ def band_filename(self, band):
return os.path.join(self.path, 'bands', str(band / 1000), str(band)) return os.path.join(self.path, 'bands', str(band / 1000), str(band))
def retrieve_data(self, band, offset, size): def retrieve_data(self, band, offset, size):
if self.band != band: if self.read_band != band:
self.band = band self.read_band = band
self.band_fd = open(self.band_filename(band), 'rb') if self.read_fd:
self.band_fd.seek(offset) self.read_fd.close()
return self.band_fd.read(size) self.read_fd = open(self.band_filename(band), 'rb')
self.read_fd.seek(offset)
return self.read_fd.read(size)
def store_data(self, data): def store_data(self, data):
if self.band_fd is None: if self.write_band is None:
self.band = self.nextband self.write_band = self.nextband
self.nextband += 1 self.nextband += 1
if self.band % 1000 == 0: if self.write_band % 1000 == 0:
os.mkdir(os.path.join(self.path, 'bands', str(self.band / 1000))) os.mkdir(os.path.join(self.path, 'bands', str(self.write_band / 1000)))
assert not os.path.exists(self.band_filename(self.band)) assert not os.path.exists(self.band_filename(self.write_band))
self.band_fd = open(self.band_filename(self.band), 'ab') self.write_fd = open(self.band_filename(self.write_band), 'ab')
offset = self.band_fd.tell() band = self.write_band
self.band_fd.write(data) offset = self.write_fd.tell()
self.write_fd.write(data)
if offset + len(data) > self.bandlimit: if offset + len(data) > self.bandlimit:
self.band_fd.close() self.write_band = None
self.band_fd = None return band, offset, len(data)
return self.band, offset, len(data)
def put(self, ns, id, data): def put(self, ns, id, data):
""" """