mirror
/
borg
mirror of https://github.com/borgbackup/borg.git
1
0
Fork 0
Code Issues Releases Wiki Activity
borg/attic/testsuite/repository.py

275 lines
11 KiB
Python
Raw Blame History

import os
import shutil
import tempfile
from attic.hashindex import NSIndex
from attic.helpers import Location, IntegrityError
from attic.remote import RemoteRepository
from attic.repository import Repository
from attic.testsuite import AtticTestCase
class RepositoryTestCase(AtticTestCase):
def open(self, create=False):
return Repository(os.path.join(self.tmppath, 'repository'), create=create)
def setUp(self):
self.tmppath = tempfile.mkdtemp()
self.repository = self.open(create=True)
def tearDown(self):
self.repository.close()
shutil.rmtree(self.tmppath)
def test1(self):
for x in range(100):
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
key50 = ('%-32d' % 50).encode('ascii')
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
self.repository.delete(key50)
self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(key50))
self.repository.commit()
self.repository.close()
repository2 = self.open()
self.assert_raises(Repository.DoesNotExist, lambda: repository2.get(key50))
for x in range(100):
if x == 50:
continue
self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA')
repository2.close()
def test2(self):
"""Test multiple sequential transactions
"""
self.repository.put(b'00000000000000000000000000000000', b'foo')
self.repository.put(b'00000000000000000000000000000001', b'foo')
self.repository.commit()
self.repository.delete(b'00000000000000000000000000000000')
self.repository.put(b'00000000000000000000000000000001', b'bar')
self.repository.commit()
self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar')
def test_consistency(self):
"""Test cache consistency
"""
self.repository.put(b'00000000000000000000000000000000', b'foo')
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
self.repository.put(b'00000000000000000000000000000000', b'foo2')
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
self.repository.put(b'00000000000000000000000000000000', b'bar')
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar')
self.repository.delete(b'00000000000000000000000000000000')
self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(b'00000000000000000000000000000000'))
def test_consistency2(self):
"""Test cache consistency2
"""
self.repository.put(b'00000000000000000000000000000000', b'foo')
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
self.repository.commit()
self.repository.put(b'00000000000000000000000000000000', b'foo2')
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
self.repository.rollback()
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
def test_single_kind_transactions(self):
# put
self.repository.put(b'00000000000000000000000000000000', b'foo')
self.repository.commit()
self.repository.close()
# replace
self.repository = self.open()
self.repository.put(b'00000000000000000000000000000000', b'bar')
self.repository.commit()
self.repository.close()
# delete
self.repository = self.open()
self.repository.delete(b'00000000000000000000000000000000')
self.repository.commit()
def test_list(self):
for x in range(100):
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
all = self.repository.list()
self.assert_equal(len(all), 100)
first_half = self.repository.list(limit=50)
self.assert_equal(len(first_half), 50)
self.assert_equal(first_half, all[:50])
second_half = self.repository.list(marker=first_half[-1])
self.assert_equal(len(second_half), 50)
self.assert_equal(second_half, all[50:])
self.assert_equal(len(self.repository.list(limit=50)), 50)
class RepositoryCheckTestCase(AtticTestCase):
def open(self, create=False):
return Repository(os.path.join(self.tmppath, 'repository'), create=create)
def reopen(self):
if self.repository:
self.repository.close()
self.repository = self.open()
def setUp(self):
self.tmppath = tempfile.mkdtemp()
self.repository = self.open(create=True)
def tearDown(self):
self.repository.close()
shutil.rmtree(self.tmppath)
def list_indices(self):
return [name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if name.startswith('index.')]
def check(self, repair=False, status=True):
self.assert_equal(self.repository.check(repair=repair), status)
# Make sure no tmp files are left behind
self.assert_equal([name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if 'tmp' in name], [], 'Found tmp files')
def get_objects(self, *ids):
for id_ in ids:
self.repository.get(('%032d' % id_).encode('ascii'))
def add_objects(self, segments):
for ids in segments:
for id_ in ids:
self.repository.put(('%032d' % id_).encode('ascii'), b'data')
self.repository.commit()
def get_head(self):
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
def open_index(self):
return NSIndex(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
def corrupt_object(self, id_):
idx = self.open_index()
segment, offset = idx[('%032d' % id_).encode('ascii')]
with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
fd.seek(offset)
fd.write(b'BOOM')
def delete_segment(self, segment):
os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)))
def delete_index(self):
os.unlink(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
def rename_index(self, new_name):
os.rename(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())),
os.path.join(self.tmppath, 'repository', new_name))
def list_objects(self):
return set((int(key) for key, _ in list(self.open_index().iteritems())))
def test_repair_corrupted_segment(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
self.check(status=True)
self.corrupt_object(5)
self.assert_raises(IntegrityError, lambda: self.get_objects(5))
self.repository.rollback()
# Make sure a regular check does not repair anything
self.check(status=False)
self.check(status=False)
# Make sure a repair actually repairs the repo
self.check(repair=True, status=True)
self.get_objects(4)
self.check(status=True)
self.assert_equal(set([1, 2, 3, 4, 6]), self.list_objects())
def test_repair_missing_segment(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
self.check(status=True)
self.delete_segment(1)
self.repository.rollback()
self.check(repair=True, status=True)
self.assert_equal(set([1, 2, 3]), self.list_objects())
def test_repair_missing_commit_segment(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.delete_segment(1)
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(repair=True, status=True)
self.assert_raises(Repository.DoesNotExist, lambda: self.get_objects(4))
self.assert_equal(set([1, 2, 3]), self.list_objects())
def test_repair_corrupted_commit_segment(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd:
fd.seek(-1, os.SEEK_END)
fd.write(b'X')
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(repair=True, status=True)
self.get_objects(3)
self.assert_raises(Repository.DoesNotExist, lambda: self.get_objects(4))
self.assert_equal(set([1, 2, 3]), self.list_objects())
def test_repair_no_commits(self):
self.add_objects([[1, 2, 3]])
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '0'), 'r+b') as fd:
fd.seek(-1, os.SEEK_END)
fd.write(b'X')
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(status=False)
self.assert_equal(self.list_indices(), ['index.0'])
self.check(repair=True, status=True)
self.assert_equal(self.list_indices(), ['index.1'])
self.check(status=True)
self.get_objects(3)
self.assert_equal(set([1, 2, 3]), self.list_objects())
def test_repair_missing_index(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.delete_index()
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(repair=True, status=True)
self.check(status=True)
self.get_objects(4)
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
def test_repair_index_too_old(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.assert_equal(self.list_indices(), ['index.1'])
self.rename_index('index.0')
self.assert_equal(self.list_indices(), ['index.0'])
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(repair=True, status=True)
self.assert_equal(self.list_indices(), ['index.1'])
self.check(status=True)
self.get_objects(4)
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
def test_repair_index_too_new(self):
self.add_objects([[1, 2, 3], [4, 5, 6]])
self.assert_equal(self.list_indices(), ['index.1'])
self.rename_index('index.100')
self.assert_equal(self.list_indices(), ['index.100'])
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
self.check(status=False)
self.check(repair=True, status=True)
self.assert_equal(self.list_indices(), ['index.1'])
self.check(status=True)
self.get_objects(4)
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
class RemoteRepositoryTestCase(RepositoryTestCase):
def open(self, create=False):
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
def open(self, create=False):
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
Reference in New Issue View Git Blame Copy Permalink