mirror of
https://github.com/borgbackup/borg.git
synced 2025-01-03 05:35:58 +00:00
Removed unused store implementations
This commit is contained in:
Jonas Borgström
parent
98b1b5e0ce
commit
cbb62fd2a6
2 changed files with 0 additions and 413 deletions
|
|
@ -1,256 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
import fcntl
|
||||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
|
||||
class Store(object):
|
||||
"""
|
||||
"""
|
||||
|
||||
class DoesNotExist(KeyError):
|
||||
""""""
|
||||
|
||||
class AlreadyExists(KeyError):
|
||||
""""""
|
||||
|
||||
IDLE = 'Idle'
|
||||
OPEN = 'Open'
|
||||
ACTIVE = 'Active'
|
||||
VERSION = 'DEDUPESTORE VERSION 1'
|
||||
|
||||
def __init__(self, path):
|
||||
self.tid = '-1'
|
||||
self.state = Store.IDLE
|
||||
if not os.path.exists(path):
|
||||
self.create(path)
|
||||
self.open(path)
|
||||
|
||||
def create(self, path):
|
||||
os.mkdir(path)
|
||||
open(os.path.join(path, 'version'), 'wb').write(self.VERSION)
|
||||
open(os.path.join(path, 'uuid'), 'wb').write(str(uuid.uuid4()))
|
||||
open(os.path.join(path, 'tid'), 'wb').write('0')
|
||||
os.mkdir(os.path.join(path, 'data'))
|
||||
|
||||
def open(self, path):
|
||||
self.path = path
|
||||
if not os.path.isdir(path):
|
||||
raise Exception('%s Does not look like a store')
|
||||
version_path = os.path.join(path, 'version')
|
||||
if not os.path.exists(version_path) or open(version_path, 'rb').read() != self.VERSION:
|
||||
raise Exception('%s Does not look like a store2')
|
||||
self.uuid = open(os.path.join(path, 'uuid'), 'rb').read()
|
||||
self.lock_fd = open(os.path.join(path, 'lock'), 'w')
|
||||
fcntl.flock(self.lock_fd, fcntl.LOCK_EX)
|
||||
self.tid = int(open(os.path.join(path, 'tid'), 'r').read())
|
||||
self.recover()
|
||||
|
||||
def recover(self):
|
||||
if os.path.exists(os.path.join(self.path, 'txn-active')):
|
||||
shutil.rmtree(os.path.join(self.path, 'txn-active'))
|
||||
if os.path.exists(os.path.join(self.path, 'txn-commit')):
|
||||
self.apply_txn()
|
||||
if os.path.exists(os.path.join(self.path, 'txn-applied')):
|
||||
shutil.rmtree(os.path.join(self.path, 'txn-applied'))
|
||||
self.state = Store.OPEN
|
||||
self.txn_delete = []
|
||||
self.txn_write = []
|
||||
|
||||
def close(self):
|
||||
self.recover()
|
||||
self.lock_fd.close()
|
||||
self.state = Store.IDLE
|
||||
|
||||
def commit(self):
|
||||
"""
|
||||
"""
|
||||
if self.state == Store.OPEN:
|
||||
return
|
||||
assert self.state == Store.ACTIVE
|
||||
with open(os.path.join(self.path, 'txn-active', 'delete_index'), 'wb') as fd:
|
||||
fd.write('\n'.join(self.txn_delete))
|
||||
with open(os.path.join(self.path, 'txn-active', 'write_index'), 'wb') as fd:
|
||||
fd.write('\n'.join(self.txn_write))
|
||||
with open(os.path.join(self.path, 'txn-active', 'tid'), 'wb') as fd:
|
||||
fd.write(str(self.tid + 1))
|
||||
os.rename(os.path.join(self.path, 'txn-active'),
|
||||
os.path.join(self.path, 'txn-commit'))
|
||||
self.recover()
|
||||
|
||||
def apply_txn(self):
|
||||
assert os.path.isdir(os.path.join(self.path, 'txn-commit'))
|
||||
tid = int(open(os.path.join(self.path, 'txn-commit', 'tid'), 'rb').read())
|
||||
assert tid == self.tid + 1
|
||||
delete_list = [line.strip() for line in
|
||||
open(os.path.join(self.path, 'txn-commit', 'delete_index'), 'rb').readlines()]
|
||||
for name in delete_list:
|
||||
path = os.path.join(self.path, 'data', name)
|
||||
os.unlink(path)
|
||||
write_list = [line.strip() for line in
|
||||
open(os.path.join(self.path, 'txn-commit', 'write_index'), 'rb').readlines()]
|
||||
for name in write_list:
|
||||
destname = os.path.join(self.path, 'data', name)
|
||||
if not os.path.exists(os.path.dirname(destname)):
|
||||
os.makedirs(os.path.dirname(destname))
|
||||
os.rename(os.path.join(self.path, 'txn-commit', 'write', name), destname)
|
||||
with open(os.path.join(self.path, 'tid'), 'wb') as fd:
|
||||
fd.write(str(tid))
|
||||
os.rename(os.path.join(self.path, 'txn-commit'),
|
||||
os.path.join(self.path, 'txn-applied'))
|
||||
shutil.rmtree(os.path.join(self.path, 'txn-applied'))
|
||||
self.tid = tid
|
||||
|
||||
def rollback(self):
|
||||
"""
|
||||
"""
|
||||
self.recover()
|
||||
|
||||
def prepare_txn(self):
|
||||
if self.state == Store.ACTIVE:
|
||||
return os.path.join(self.path, 'txn-active')
|
||||
elif self.state == Store.OPEN:
|
||||
os.mkdir(os.path.join(self.path, 'txn-active'))
|
||||
os.mkdir(os.path.join(self.path, 'txn-active', 'write'))
|
||||
self.state = Store.ACTIVE
|
||||
|
||||
def _filename(self, ns, id, base=''):
|
||||
ns = ns.encode('hex')
|
||||
id = id.encode('hex')
|
||||
return os.path.join(base, ns, id[:2], id[2:4], id[4:])
|
||||
|
||||
def get(self, ns, id):
|
||||
"""
|
||||
"""
|
||||
path = self._filename(ns, id)
|
||||
if path in self.txn_write:
|
||||
filename = os.path.join(self.path, 'txn-active', 'write', path)
|
||||
return open(filename, 'rb').read()
|
||||
if path in self.txn_delete:
|
||||
raise Store.DoesNotExist('Object %s:%s does not exist' % (ns.encode('hex'), id.encode('hex')))
|
||||
filename = self._filename(ns, id, os.path.join(self.path, 'data'))
|
||||
if os.path.exists(filename):
|
||||
return open(filename, 'rb').read()
|
||||
else:
|
||||
raise Store.DoesNotExist('Object %s:%s does not exist' % (ns.encode('hex'), id.encode('hex')))
|
||||
|
||||
def put(self, ns, id, data):
|
||||
"""
|
||||
"""
|
||||
self.prepare_txn()
|
||||
path = self._filename(ns, id)
|
||||
filename = self._filename(ns, id, os.path.join(self.path, 'data'))
|
||||
if (path in self.txn_write or
|
||||
(path not in self.txn_delete and os.path.exists(filename))):
|
||||
raise Store.AlreadyExists('Object already exists: %s:%s' % (ns.encode('hex'), id.encode('hex')))
|
||||
if path in self.txn_delete:
|
||||
self.txn_delete.remove(path)
|
||||
if path not in self.txn_write:
|
||||
self.txn_write.append(path)
|
||||
filename = self._filename(ns, id, os.path.join(self.path, 'txn-active', 'write'))
|
||||
if not os.path.exists(os.path.dirname(filename)):
|
||||
os.makedirs(os.path.dirname(filename))
|
||||
with open(filename, 'wb') as fd:
|
||||
fd.write(data)
|
||||
|
||||
def delete(self, ns, id):
|
||||
"""
|
||||
"""
|
||||
self.prepare_txn()
|
||||
path = self._filename(ns, id)
|
||||
if path in self.txn_write:
|
||||
filename = self._filename(ns, id, os.path.join(self.path, 'txn-active', 'write'))
|
||||
self.txn_write.remove(path)
|
||||
os.unlink(filename)
|
||||
else:
|
||||
filename = os.path.join(self.path, 'data', path)
|
||||
if os.path.exists(filename):
|
||||
self.txn_delete.append(path)
|
||||
else:
|
||||
raise Store.DoesNotExist('Object does not exist: %s' % hash.encode('hex'))
|
||||
|
||||
def list(self, ns, prefix='', marker=None, max_keys=1000000):
|
||||
for x in self._walker(os.path.join(self.path, 'data', ns.encode('hex')),
|
||||
prefix, marker, '', max_keys):
|
||||
yield x
|
||||
|
||||
def _walker(self, path, prefix, marker, base, max_keys):
|
||||
n = 0
|
||||
for name in sorted(os.listdir(path)):
|
||||
if n >= max_keys:
|
||||
return
|
||||
id = name.decode('hex')
|
||||
if os.path.isdir(os.path.join(path, name)):
|
||||
if prefix and not id.startswith(prefix[:len(id)]):
|
||||
continue
|
||||
for x in self.foo(os.path.join(path, name),
|
||||
prefix[len(id):], marker,
|
||||
base + id, max_keys - n):
|
||||
yield x
|
||||
n += 1
|
||||
else:
|
||||
if prefix and not id.startswith(prefix):
|
||||
continue
|
||||
if not marker or base + id >= marker:
|
||||
yield base + id
|
||||
n += 1
|
||||
|
||||
|
||||
class StoreTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmppath = tempfile.mkdtemp()
|
||||
self.store = Store(os.path.join(self.tmppath, 'store'))
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmppath)
|
||||
|
||||
def test1(self):
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
self.assertEqual(self.store.state, Store.OPEN)
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.assertRaises(Store.AlreadyExists, lambda: self.store.put('SOMENS', 'SOMEID', 'SOMEDATA'))
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.rollback()
|
||||
self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
|
||||
def test2(self):
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
self.assertEqual(self.store.state, Store.OPEN)
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.commit()
|
||||
self.assertEqual(self.store.tid, 1)
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.delete('SOMENS', 'SOMEID')
|
||||
self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.store.rollback()
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.delete('SOMENS', 'SOMEID')
|
||||
self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.store.commit()
|
||||
self.assertEqual(self.store.tid, 2)
|
||||
self.assertRaises(Store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
|
||||
def test_list(self):
|
||||
self.store.put('SOMENS', 'SOMEID12', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID1', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID123', 'SOMEDATA')
|
||||
self.store.commit()
|
||||
self.assertEqual(list(self.store.list('SOMENS', max_keys=3)),
|
||||
['SOMEID', 'SOMEID1', 'SOMEID12'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', marker='SOMEID12')),
|
||||
['SOMEID12', 'SOMEID123'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', prefix='SOMEID1', max_keys=2)),
|
||||
['SOMEID1', 'SOMEID12'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', prefix='SOMEID1', marker='SOMEID12')),
|
||||
['SOMEID12', 'SOMEID123'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
@ -1,157 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
import sqlite3
|
||||
import uuid
|
||||
|
||||
|
||||
class SqliteStore(object):
|
||||
"""
|
||||
"""
|
||||
|
||||
class DoesNotExist(KeyError):
|
||||
""""""
|
||||
|
||||
class AlreadyExists(KeyError):
|
||||
""""""
|
||||
|
||||
IDLE = 'Idle'
|
||||
OPEN = 'Open'
|
||||
ACTIVE = 'Active'
|
||||
VERSION = 'DEDUPESTORE VERSION 1'
|
||||
|
||||
def __init__(self, path):
|
||||
if not os.path.exists(path):
|
||||
self.create(path)
|
||||
self.cnx = sqlite3.connect(path)
|
||||
self.cursor = self.cnx.cursor()
|
||||
self.uuid, self.tid = self.cursor.execute('SELECT uuid, tid FROM system').fetchone()
|
||||
self.state = self.OPEN
|
||||
|
||||
def create(self, path):
|
||||
cnx = sqlite3.connect(path)
|
||||
cnx.execute('PRAGMA auto_vacuum=full')
|
||||
cnx.execute('CREATE TABLE objects(ns TEXT NOT NULL, id NOT NULL, data NOT NULL)')
|
||||
cnx.execute('CREATE TABLE system(uuid NOT NULL, tid NOT NULL)')
|
||||
cnx.execute('INSERT INTO system VALUES(?,?)', (uuid.uuid1().hex, 0))
|
||||
cnx.execute('CREATE UNIQUE INDEX objects_pk ON objects(ns, id)')
|
||||
|
||||
def close(self):
|
||||
self.cnx.close()
|
||||
|
||||
def commit(self):
|
||||
"""
|
||||
"""
|
||||
self.cursor.execute('UPDATE system SET tid=tid+1')
|
||||
import time
|
||||
t = time.time()
|
||||
self.cnx.commit()
|
||||
print time.time() - t
|
||||
self.tid += 1
|
||||
|
||||
def rollback(self):
|
||||
"""
|
||||
"""
|
||||
self.cnx.rollback()
|
||||
|
||||
def get(self, ns, id):
|
||||
"""
|
||||
"""
|
||||
self.cursor.execute('SELECT data FROM objects WHERE ns=? and id=?',
|
||||
(ns.encode('hex'), id.encode('hex')))
|
||||
row = self.cursor.fetchone()
|
||||
if row:
|
||||
return str(row[0])
|
||||
else:
|
||||
raise self.DoesNotExist
|
||||
|
||||
def put(self, ns, id, data):
|
||||
"""
|
||||
"""
|
||||
try:
|
||||
self.cursor.execute('INSERT INTO objects (ns, id, data) '
|
||||
'VALUES(?, ?, ?)',
|
||||
(ns.encode('hex'), id.encode('hex'),
|
||||
sqlite3.Binary(data)))
|
||||
except sqlite3.IntegrityError:
|
||||
raise self.AlreadyExists
|
||||
|
||||
def delete(self, ns, id):
|
||||
"""
|
||||
"""
|
||||
self.cursor.execute('DELETE FROM objects WHERE ns=? AND id=?',
|
||||
(ns.encode('hex'), id.encode('hex')))
|
||||
|
||||
def list(self, ns, prefix='', marker=None, max_keys=1000000):
|
||||
"""
|
||||
"""
|
||||
condition = ''
|
||||
if prefix:
|
||||
condition += ' AND id LIKE :prefix'
|
||||
if marker:
|
||||
condition += ' AND id >= :marker'
|
||||
args = dict(ns=ns.encode('hex'), prefix=prefix.encode('hex') + '%',
|
||||
marker=marker and marker.encode('hex'))
|
||||
for row in self.cursor.execute('SELECT id FROM objects WHERE '
|
||||
'ns=:ns ' + condition + ' LIMIT ' + str(max_keys),
|
||||
args):
|
||||
yield row[0].decode('hex')
|
||||
|
||||
|
||||
class SqliteStoreTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmppath = tempfile.mkdtemp()
|
||||
self.store = SqliteStore(os.path.join(self.tmppath, 'store'))
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.tmppath)
|
||||
|
||||
def test1(self):
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
self.assertEqual(self.store.state, self.store.OPEN)
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.assertRaises(self.store.AlreadyExists, lambda: self.store.put('SOMENS', 'SOMEID', 'SOMEDATA'))
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.rollback()
|
||||
self.assertRaises(self.store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
|
||||
def test2(self):
|
||||
self.assertEqual(self.store.tid, 0)
|
||||
self.assertEqual(self.store.state, self.store.OPEN)
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.commit()
|
||||
self.assertEqual(self.store.tid, 1)
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.delete('SOMENS', 'SOMEID')
|
||||
self.assertRaises(self.store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.store.rollback()
|
||||
self.assertEqual(self.store.get('SOMENS', 'SOMEID'), 'SOMEDATA')
|
||||
self.store.delete('SOMENS', 'SOMEID')
|
||||
self.assertRaises(self.store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
self.store.commit()
|
||||
self.assertEqual(self.store.tid, 2)
|
||||
self.assertRaises(self.store.DoesNotExist, lambda: self.store.get('SOMENS', 'SOMEID'))
|
||||
|
||||
def test_list(self):
|
||||
self.store.put('SOMENS', 'SOMEID12', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID1', 'SOMEDATA')
|
||||
self.store.put('SOMENS', 'SOMEID123', 'SOMEDATA')
|
||||
self.store.commit()
|
||||
self.assertEqual(list(self.store.list('SOMENS', max_keys=3)),
|
||||
['SOMEID', 'SOMEID1', 'SOMEID12'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', marker='SOMEID12')),
|
||||
['SOMEID12', 'SOMEID123'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', prefix='SOMEID1', max_keys=2)),
|
||||
['SOMEID1', 'SOMEID12'])
|
||||
self.assertEqual(list(self.store.list('SOMENS', prefix='SOMEID1', marker='SOMEID12')),
|
||||
['SOMEID12', 'SOMEID123'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Loading…
Reference in a new issue