mirror of
https://github.com/borgbackup/borg.git
synced 2024-12-25 17:27:31 +00:00
64cd6632a1
Explicitly use write locks, instead of read locks (that are later upgraded) unless we know we will never modify the repository
326 lines
13 KiB
Python
326 lines
13 KiB
Python
import os
|
|
import shutil
|
|
import tempfile
|
|
from attic.testsuite.mock import patch
|
|
from attic.hashindex import NSIndex
|
|
from attic.helpers import Location, IntegrityError, UpgradableLock
|
|
from attic.remote import RemoteRepository
|
|
from attic.repository import Repository
|
|
from attic.testsuite import AtticTestCase
|
|
|
|
|
|
class RepositoryTestCaseBase(AtticTestCase):
|
|
|
|
def open(self, create=False):
|
|
return Repository(os.path.join(self.tmppath, 'repository'), create=create)
|
|
|
|
def setUp(self):
|
|
self.tmppath = tempfile.mkdtemp()
|
|
self.repository = self.open(create=True)
|
|
|
|
def tearDown(self):
|
|
self.repository.close()
|
|
shutil.rmtree(self.tmppath)
|
|
|
|
def reopen(self):
|
|
if self.repository:
|
|
self.repository.close()
|
|
self.repository = self.open()
|
|
|
|
|
|
class RepositoryTestCase(RepositoryTestCaseBase):
|
|
|
|
def test1(self):
|
|
for x in range(100):
|
|
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
|
|
key50 = ('%-32d' % 50).encode('ascii')
|
|
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
|
|
self.repository.delete(key50)
|
|
self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(key50))
|
|
self.repository.commit()
|
|
self.repository.close()
|
|
repository2 = self.open()
|
|
self.assert_raises(Repository.DoesNotExist, lambda: repository2.get(key50))
|
|
for x in range(100):
|
|
if x == 50:
|
|
continue
|
|
self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA')
|
|
repository2.close()
|
|
|
|
def test2(self):
|
|
"""Test multiple sequential transactions
|
|
"""
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.repository.put(b'00000000000000000000000000000001', b'foo')
|
|
self.repository.commit()
|
|
self.repository.delete(b'00000000000000000000000000000000')
|
|
self.repository.put(b'00000000000000000000000000000001', b'bar')
|
|
self.repository.commit()
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar')
|
|
|
|
def test_consistency(self):
|
|
"""Test cache consistency
|
|
"""
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
|
self.repository.put(b'00000000000000000000000000000000', b'bar')
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar')
|
|
self.repository.delete(b'00000000000000000000000000000000')
|
|
self.assert_raises(Repository.DoesNotExist, lambda: self.repository.get(b'00000000000000000000000000000000'))
|
|
|
|
def test_consistency2(self):
|
|
"""Test cache consistency2
|
|
"""
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
|
self.repository.commit()
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
|
self.repository.rollback()
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
|
|
|
def test_overwrite_in_same_transaction(self):
|
|
"""Test cache consistency2
|
|
"""
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
|
self.repository.commit()
|
|
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
|
|
|
def test_single_kind_transactions(self):
|
|
# put
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.repository.commit()
|
|
self.repository.close()
|
|
# replace
|
|
self.repository = self.open()
|
|
self.repository.put(b'00000000000000000000000000000000', b'bar')
|
|
self.repository.commit()
|
|
self.repository.close()
|
|
# delete
|
|
self.repository = self.open()
|
|
self.repository.delete(b'00000000000000000000000000000000')
|
|
self.repository.commit()
|
|
|
|
def test_list(self):
|
|
for x in range(100):
|
|
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
|
|
all = self.repository.list()
|
|
self.assert_equal(len(all), 100)
|
|
first_half = self.repository.list(limit=50)
|
|
self.assert_equal(len(first_half), 50)
|
|
self.assert_equal(first_half, all[:50])
|
|
second_half = self.repository.list(marker=first_half[-1])
|
|
self.assert_equal(len(second_half), 50)
|
|
self.assert_equal(second_half, all[50:])
|
|
self.assert_equal(len(self.repository.list(limit=50)), 50)
|
|
|
|
|
|
class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
|
|
|
def add_keys(self):
|
|
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
|
self.repository.put(b'00000000000000000000000000000001', b'bar')
|
|
self.repository.put(b'00000000000000000000000000000003', b'bar')
|
|
self.repository.commit()
|
|
self.repository.put(b'00000000000000000000000000000001', b'bar2')
|
|
self.repository.put(b'00000000000000000000000000000002', b'boo')
|
|
self.repository.delete(b'00000000000000000000000000000003')
|
|
|
|
def test_replay_of_missing_index(self):
|
|
self.add_keys()
|
|
for name in os.listdir(self.repository.path):
|
|
if name.startswith('index.'):
|
|
os.unlink(os.path.join(self.repository.path, name))
|
|
self.reopen()
|
|
self.assert_equal(len(self.repository), 3)
|
|
self.assert_equal(self.repository.check(), True)
|
|
|
|
def test_crash_before_compact_segments(self):
|
|
self.add_keys()
|
|
self.repository.compact_segments = None
|
|
try:
|
|
self.repository.commit()
|
|
except TypeError:
|
|
pass
|
|
self.reopen()
|
|
self.assert_equal(len(self.repository), 3)
|
|
self.assert_equal(self.repository.check(), True)
|
|
|
|
def test_replay_of_readonly_repository(self):
|
|
self.add_keys()
|
|
for name in os.listdir(self.repository.path):
|
|
if name.startswith('index.'):
|
|
os.unlink(os.path.join(self.repository.path, name))
|
|
with patch.object(UpgradableLock, 'upgrade', side_effect=UpgradableLock.WriteLockFailed) as upgrade:
|
|
self.reopen()
|
|
self.assert_raises(UpgradableLock.WriteLockFailed, lambda: len(self.repository))
|
|
upgrade.assert_called_once()
|
|
|
|
|
|
def test_crash_before_write_index(self):
|
|
self.add_keys()
|
|
self.repository.write_index = None
|
|
try:
|
|
self.repository.commit()
|
|
except TypeError:
|
|
pass
|
|
self.reopen()
|
|
self.assert_equal(len(self.repository), 3)
|
|
self.assert_equal(self.repository.check(), True)
|
|
|
|
def test_crash_before_deleting_compacted_segments(self):
|
|
self.add_keys()
|
|
self.repository.io.delete_segment = None
|
|
try:
|
|
self.repository.commit()
|
|
except TypeError:
|
|
pass
|
|
self.reopen()
|
|
self.assert_equal(len(self.repository), 3)
|
|
self.assert_equal(self.repository.check(), True)
|
|
self.assert_equal(len(self.repository), 3)
|
|
|
|
|
|
class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
|
|
|
def list_indices(self):
|
|
return [name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if name.startswith('index.')]
|
|
|
|
def check(self, repair=False, status=True):
|
|
self.assert_equal(self.repository.check(repair=repair), status)
|
|
# Make sure no tmp files are left behind
|
|
self.assert_equal([name for name in os.listdir(os.path.join(self.tmppath, 'repository')) if 'tmp' in name], [], 'Found tmp files')
|
|
|
|
def get_objects(self, *ids):
|
|
for id_ in ids:
|
|
self.repository.get(('%032d' % id_).encode('ascii'))
|
|
|
|
def add_objects(self, segments):
|
|
for ids in segments:
|
|
for id_ in ids:
|
|
self.repository.put(('%032d' % id_).encode('ascii'), b'data')
|
|
self.repository.commit()
|
|
|
|
def get_head(self):
|
|
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
|
|
|
|
def open_index(self):
|
|
return NSIndex(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
|
|
|
|
def corrupt_object(self, id_):
|
|
idx = self.open_index()
|
|
segment, offset = idx[('%032d' % id_).encode('ascii')]
|
|
with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
|
|
fd.seek(offset)
|
|
fd.write(b'BOOM')
|
|
|
|
def delete_segment(self, segment):
|
|
os.unlink(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)))
|
|
|
|
def delete_index(self):
|
|
os.unlink(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())))
|
|
|
|
def rename_index(self, new_name):
|
|
os.rename(os.path.join(self.tmppath, 'repository', 'index.{}'.format(self.get_head())),
|
|
os.path.join(self.tmppath, 'repository', new_name))
|
|
|
|
def list_objects(self):
|
|
return set(int(key) for key in self.repository.list())
|
|
|
|
def test_repair_corrupted_segment(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
|
|
self.check(status=True)
|
|
self.corrupt_object(5)
|
|
self.assert_raises(IntegrityError, lambda: self.get_objects(5))
|
|
self.repository.rollback()
|
|
# Make sure a regular check does not repair anything
|
|
self.check(status=False)
|
|
self.check(status=False)
|
|
# Make sure a repair actually repairs the repo
|
|
self.check(repair=True, status=True)
|
|
self.get_objects(4)
|
|
self.check(status=True)
|
|
self.assert_equal(set([1, 2, 3, 4, 6]), self.list_objects())
|
|
|
|
def test_repair_missing_segment(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
|
|
self.check(status=True)
|
|
self.delete_segment(1)
|
|
self.repository.rollback()
|
|
self.check(repair=True, status=True)
|
|
self.assert_equal(set([1, 2, 3]), self.list_objects())
|
|
|
|
def test_repair_missing_commit_segment(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
self.delete_segment(1)
|
|
self.assert_raises(Repository.DoesNotExist, lambda: self.get_objects(4))
|
|
self.assert_equal(set([1, 2, 3]), self.list_objects())
|
|
|
|
def test_repair_corrupted_commit_segment(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '1'), 'r+b') as fd:
|
|
fd.seek(-1, os.SEEK_END)
|
|
fd.write(b'X')
|
|
self.assert_raises(Repository.DoesNotExist, lambda: self.get_objects(4))
|
|
self.check(status=True)
|
|
self.get_objects(3)
|
|
self.assert_equal(set([1, 2, 3]), self.list_objects())
|
|
|
|
def test_repair_no_commits(self):
|
|
self.add_objects([[1, 2, 3]])
|
|
with open(os.path.join(self.tmppath, 'repository', 'data', '0', '0'), 'r+b') as fd:
|
|
fd.seek(-1, os.SEEK_END)
|
|
fd.write(b'X')
|
|
self.assert_raises(Repository.CheckNeeded, lambda: self.get_objects(4))
|
|
self.check(status=False)
|
|
self.check(status=False)
|
|
self.assert_equal(self.list_indices(), ['index.0'])
|
|
self.check(repair=True, status=True)
|
|
self.assert_equal(self.list_indices(), ['index.1'])
|
|
self.check(status=True)
|
|
self.get_objects(3)
|
|
self.assert_equal(set([1, 2, 3]), self.list_objects())
|
|
|
|
def test_repair_missing_index(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
self.delete_index()
|
|
self.check(status=True)
|
|
self.get_objects(4)
|
|
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
|
|
|
|
def test_repair_index_too_new(self):
|
|
self.add_objects([[1, 2, 3], [4, 5, 6]])
|
|
self.assert_equal(self.list_indices(), ['index.1'])
|
|
self.rename_index('index.100')
|
|
self.check(status=True)
|
|
self.assert_equal(self.list_indices(), ['index.1'])
|
|
self.get_objects(4)
|
|
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
|
|
|
|
def test_crash_before_compact(self):
|
|
self.repository.put(bytes(32), b'data')
|
|
self.repository.put(bytes(32), b'data2')
|
|
# Simulate a crash before compact
|
|
with patch.object(Repository, 'compact_segments') as compact:
|
|
self.repository.commit()
|
|
compact.assert_called_once()
|
|
self.reopen()
|
|
self.check(repair=True)
|
|
self.assert_equal(self.repository.get(bytes(32)), b'data2')
|
|
|
|
|
|
class RemoteRepositoryTestCase(RepositoryTestCase):
|
|
|
|
def open(self, create=False):
|
|
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
|
|
|
|
|
|
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):
|
|
|
|
def open(self, create=False):
|
|
return RemoteRepository(Location('__testsuite__:' + os.path.join(self.tmppath, 'repository')), create=create)
|