mirror of https://github.com/borgbackup/borg.git
Merge pull request #1535 from ThomasWaldmann/merge-1.0-maint
Merge 1.0 maint
This commit is contained in:
TW
GitHub
commit
168ce1dc86
|
|
@ -13,7 +13,6 @@
|
|||
border-color: #000000;
|
||||
}
|
||||
|
||||
|
||||
.wy-side-nav-search > a {
|
||||
color: rgba(255, 255, 255, 0.5);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,6 +42,14 @@ class InvalidRPCMethod(Error):
|
|||
"""RPC method {} is not valid"""
|
||||
|
||||
|
||||
class UnexpectedRPCDataFormatFromClient(Error):
|
||||
"""Borg {}: Got unexpected RPC data format from client."""
|
||||
|
||||
|
||||
class UnexpectedRPCDataFormatFromServer(Error):
|
||||
"""Got unexpected RPC data format from server."""
|
||||
|
||||
|
||||
class RepositoryServer: # pragma: no cover
|
||||
rpc_methods = (
|
||||
'__len__',
|
||||
|
|
@ -84,13 +92,18 @@ class RepositoryServer: # pragma: no cover
|
|||
if r:
|
||||
data = os.read(stdin_fd, BUFSIZE)
|
||||
if not data:
|
||||
self.repository.close()
|
||||
if self.repository is not None:
|
||||
self.repository.close()
|
||||
else:
|
||||
os.write(stderr_fd, "Borg {}: Got connection close before repository was opened.\n"
|
||||
.format(__version__).encode())
|
||||
return
|
||||
unpacker.feed(data)
|
||||
for unpacked in unpacker:
|
||||
if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
|
||||
self.repository.close()
|
||||
raise Exception("Unexpected RPC data format.")
|
||||
if self.repository is not None:
|
||||
self.repository.close()
|
||||
raise UnexpectedRPCDataFormatFromClient(__version__)
|
||||
type, msgid, method, args = unpacked
|
||||
method = method.decode('ascii')
|
||||
try:
|
||||
|
|
@ -354,7 +367,7 @@ This problem will go away as soon as the server has been upgraded to 1.0.7+.
|
|||
self.unpacker.feed(data)
|
||||
for unpacked in self.unpacker:
|
||||
if not (isinstance(unpacked, tuple) and len(unpacked) == 4):
|
||||
raise Exception("Unexpected RPC data format.")
|
||||
raise UnexpectedRPCDataFormatFromServer()
|
||||
type, msgid, error, res = unpacked
|
||||
if msgid in self.ignore_responses:
|
||||
self.ignore_responses.remove(msgid)
|
||||
|
|
|
|||
|
|
@ -285,8 +285,14 @@ class ArchiverTestCaseBase(BaseTestCase):
|
|||
# Symlink
|
||||
if are_symlinks_supported():
|
||||
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
|
||||
if xattr.is_enabled(self.input_path):
|
||||
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
|
||||
self.create_regular_file('fusexattr', size=1)
|
||||
if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
|
||||
# ironically, due to the way how fakeroot works, comparing fuse file xattrs to orig file xattrs
|
||||
# will FAIL if fakeroot supports xattrs, thus we only set the xattr if XATTR_FAKEROOT is False.
|
||||
# This is because fakeroot with xattr-support does not propagate xattrs of the underlying file
|
||||
# into "fakeroot space". Because the xattrs exposed by borgfs are these of an underlying file
|
||||
# (from fakeroots point of view) they are invisible to the test process inside the fakeroot.
|
||||
xattr.setxattr(os.path.join(self.input_path, 'fusexattr'), 'user.foo', b'bar')
|
||||
# XXX this always fails for me
|
||||
# ubuntu 14.04, on a TMP dir filesystem with user_xattr, using fakeroot
|
||||
# same for newer ubuntu and centos.
|
||||
|
|
@ -362,7 +368,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
|
|||
self.assert_in(name, list_output)
|
||||
self.assert_dirs_equal('input', 'output/input')
|
||||
info_output = self.cmd('info', self.repository_location + '::test')
|
||||
item_count = 3 if has_lchflags else 4 # one file is UF_NODUMP
|
||||
item_count = 4 if has_lchflags else 5 # one file is UF_NODUMP
|
||||
self.assert_in('Number of files: %d' % item_count, info_output)
|
||||
shutil.rmtree(self.cache_path)
|
||||
with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'):
|
||||
|
|
@ -1399,7 +1405,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
|
|||
with open(in_fn, 'rb') as in_f, open(out_fn, 'rb') as out_f:
|
||||
assert in_f.read() == out_f.read()
|
||||
# list/read xattrs
|
||||
if xattr.is_enabled(self.input_path):
|
||||
in_fn = 'input/fusexattr'
|
||||
out_fn = os.path.join(mountpoint, 'input', 'fusexattr')
|
||||
if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path):
|
||||
assert xattr.listxattr(out_fn) == ['user.foo', ]
|
||||
assert xattr.getxattr(out_fn, 'user.foo') == b'bar'
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -24,19 +24,19 @@ class HashIndexTestCase(BaseTestCase):
|
|||
self.assert_equal(len(idx), 0)
|
||||
# Test set
|
||||
for x in range(100):
|
||||
idx[bytes('%-32d' % x, 'ascii')] = make_value(x)
|
||||
idx[H(x)] = make_value(x)
|
||||
self.assert_equal(len(idx), 100)
|
||||
for x in range(100):
|
||||
self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x))
|
||||
self.assert_equal(idx[H(x)], make_value(x))
|
||||
# Test update
|
||||
for x in range(100):
|
||||
idx[bytes('%-32d' % x, 'ascii')] = make_value(x * 2)
|
||||
idx[H(x)] = make_value(x * 2)
|
||||
self.assert_equal(len(idx), 100)
|
||||
for x in range(100):
|
||||
self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2))
|
||||
self.assert_equal(idx[H(x)], make_value(x * 2))
|
||||
# Test delete
|
||||
for x in range(50):
|
||||
del idx[bytes('%-32d' % x, 'ascii')]
|
||||
del idx[H(x)]
|
||||
self.assert_equal(len(idx), 50)
|
||||
idx_name = tempfile.NamedTemporaryFile()
|
||||
idx.write(idx_name.name)
|
||||
|
|
@ -48,7 +48,7 @@ class HashIndexTestCase(BaseTestCase):
|
|||
idx = cls.read(idx_name.name)
|
||||
self.assert_equal(len(idx), 50)
|
||||
for x in range(50, 100):
|
||||
self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2))
|
||||
self.assert_equal(idx[H(x)], make_value(x * 2))
|
||||
idx.clear()
|
||||
self.assert_equal(len(idx), 0)
|
||||
idx.write(idx_name.name)
|
||||
|
|
@ -57,11 +57,11 @@ class HashIndexTestCase(BaseTestCase):
|
|||
|
||||
def test_nsindex(self):
|
||||
self._generic_test(NSIndex, lambda x: (x, x),
|
||||
'80fba5b40f8cf12f1486f1ba33c9d852fb2b41a5b5961d3b9d1228cf2aa9c4c9')
|
||||
'b96ec1ddabb4278cc92261ee171f7efc979dc19397cc5e89b778f05fa25bf93f')
|
||||
|
||||
def test_chunkindex(self):
|
||||
self._generic_test(ChunkIndex, lambda x: (x, x, x),
|
||||
'1d71865e72e3c3af18d3c7216b6fa7b014695eaa3ed7f14cf9cd02fba75d1c95')
|
||||
'9d437a1e145beccc790c69e66ba94fc17bd982d83a401c9c6e524609405529d8')
|
||||
|
||||
def test_resize(self):
|
||||
n = 2000 # Must be >= MIN_BUCKETS
|
||||
|
|
@ -71,11 +71,11 @@ class HashIndexTestCase(BaseTestCase):
|
|||
initial_size = os.path.getsize(idx_name.name)
|
||||
self.assert_equal(len(idx), 0)
|
||||
for x in range(n):
|
||||
idx[bytes('%-32d' % x, 'ascii')] = x, x
|
||||
idx[H(x)] = x, x
|
||||
idx.write(idx_name.name)
|
||||
self.assert_true(initial_size < os.path.getsize(idx_name.name))
|
||||
for x in range(n):
|
||||
del idx[bytes('%-32d' % x, 'ascii')]
|
||||
del idx[H(x)]
|
||||
self.assert_equal(len(idx), 0)
|
||||
idx.write(idx_name.name)
|
||||
self.assert_equal(initial_size, os.path.getsize(idx_name.name))
|
||||
|
|
@ -83,7 +83,7 @@ class HashIndexTestCase(BaseTestCase):
|
|||
def test_iteritems(self):
|
||||
idx = NSIndex()
|
||||
for x in range(100):
|
||||
idx[bytes('%-0.32d' % x, 'ascii')] = x, x
|
||||
idx[H(x)] = x, x
|
||||
all = list(idx.iteritems())
|
||||
self.assert_equal(len(all), 100)
|
||||
second_half = list(idx.iteritems(marker=all[49][0]))
|
||||
|
|
|
|||
|
|
@ -45,21 +45,21 @@ class RepositoryTestCaseBase(BaseTestCase):
|
|||
self.repository = self.open(exclusive=exclusive)
|
||||
|
||||
def add_keys(self):
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000001', b'bar')
|
||||
self.repository.put(b'00000000000000000000000000000003', b'bar')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.put(H(3), b'bar')
|
||||
self.repository.commit()
|
||||
self.repository.put(b'00000000000000000000000000000001', b'bar2')
|
||||
self.repository.put(b'00000000000000000000000000000002', b'boo')
|
||||
self.repository.delete(b'00000000000000000000000000000003')
|
||||
self.repository.put(H(1), b'bar2')
|
||||
self.repository.put(H(2), b'boo')
|
||||
self.repository.delete(H(3))
|
||||
|
||||
|
||||
class RepositoryTestCase(RepositoryTestCaseBase):
|
||||
|
||||
def test1(self):
|
||||
for x in range(100):
|
||||
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
|
||||
key50 = ('%-32d' % 50).encode('ascii')
|
||||
self.repository.put(H(x), b'SOMEDATA')
|
||||
key50 = H(50)
|
||||
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
|
||||
self.repository.delete(key50)
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
|
||||
|
|
@ -70,69 +70,69 @@ class RepositoryTestCase(RepositoryTestCaseBase):
|
|||
for x in range(100):
|
||||
if x == 50:
|
||||
continue
|
||||
self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA')
|
||||
self.assert_equal(repository2.get(H(x)), b'SOMEDATA')
|
||||
|
||||
def test2(self):
|
||||
"""Test multiple sequential transactions
|
||||
"""
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000001', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'foo')
|
||||
self.repository.commit()
|
||||
self.repository.delete(b'00000000000000000000000000000000')
|
||||
self.repository.put(b'00000000000000000000000000000001', b'bar')
|
||||
self.repository.delete(H(0))
|
||||
self.repository.put(H(1), b'bar')
|
||||
self.repository.commit()
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar')
|
||||
self.assert_equal(self.repository.get(H(1)), b'bar')
|
||||
|
||||
def test_consistency(self):
|
||||
"""Test cache consistency
|
||||
"""
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
||||
self.repository.put(b'00000000000000000000000000000000', b'bar')
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar')
|
||||
self.repository.delete(b'00000000000000000000000000000000')
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(b'00000000000000000000000000000000'))
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
self.repository.put(H(0), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
self.repository.put(H(0), b'bar')
|
||||
self.assert_equal(self.repository.get(H(0)), b'bar')
|
||||
self.repository.delete(H(0))
|
||||
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0)))
|
||||
|
||||
def test_consistency2(self):
|
||||
"""Test cache consistency2
|
||||
"""
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
self.repository.commit()
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
||||
self.repository.put(H(0), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
self.repository.rollback()
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo')
|
||||
|
||||
def test_overwrite_in_same_transaction(self):
|
||||
"""Test cache consistency2
|
||||
"""
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo2')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(0), b'foo2')
|
||||
self.repository.commit()
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'foo2')
|
||||
|
||||
def test_single_kind_transactions(self):
|
||||
# put
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.commit()
|
||||
self.repository.close()
|
||||
# replace
|
||||
self.repository = self.open()
|
||||
with self.repository:
|
||||
self.repository.put(b'00000000000000000000000000000000', b'bar')
|
||||
self.repository.put(H(0), b'bar')
|
||||
self.repository.commit()
|
||||
# delete
|
||||
self.repository = self.open()
|
||||
with self.repository:
|
||||
self.repository.delete(b'00000000000000000000000000000000')
|
||||
self.repository.delete(H(0))
|
||||
self.repository.commit()
|
||||
|
||||
def test_list(self):
|
||||
for x in range(100):
|
||||
self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA')
|
||||
self.repository.put(H(x), b'SOMEDATA')
|
||||
all = self.repository.list()
|
||||
self.assert_equal(len(all), 100)
|
||||
first_half = self.repository.list(limit=50)
|
||||
|
|
@ -145,10 +145,10 @@ class RepositoryTestCase(RepositoryTestCaseBase):
|
|||
|
||||
def test_max_data_size(self):
|
||||
max_data = b'x' * MAX_DATA_SIZE
|
||||
self.repository.put(b'00000000000000000000000000000000', max_data)
|
||||
self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), max_data)
|
||||
self.repository.put(H(0), max_data)
|
||||
self.assert_equal(self.repository.get(H(0)), max_data)
|
||||
self.assert_raises(IntegrityError,
|
||||
lambda: self.repository.put(b'00000000000000000000000000000001', max_data + b'x'))
|
||||
lambda: self.repository.put(H(1), max_data + b'x'))
|
||||
|
||||
|
||||
class LocalRepositoryTestCase(RepositoryTestCaseBase):
|
||||
|
|
@ -163,22 +163,22 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
|
|||
assert self.repository.compact[0] == 41 + 9
|
||||
|
||||
def test_sparse1(self):
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000001', b'123456789')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'123456789')
|
||||
self.repository.commit()
|
||||
self.repository.put(b'00000000000000000000000000000001', b'bar')
|
||||
self.repository.put(H(1), b'bar')
|
||||
self._assert_sparse()
|
||||
|
||||
def test_sparse2(self):
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(b'00000000000000000000000000000001', b'123456789')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.put(H(1), b'123456789')
|
||||
self.repository.commit()
|
||||
self.repository.delete(b'00000000000000000000000000000001')
|
||||
self.repository.delete(H(1))
|
||||
self._assert_sparse()
|
||||
|
||||
def test_sparse_delete(self):
|
||||
self.repository.put(b'00000000000000000000000000000000', b'1245')
|
||||
self.repository.delete(b'00000000000000000000000000000000')
|
||||
self.repository.put(H(0), b'1245')
|
||||
self.repository.delete(H(0))
|
||||
self.repository.io._write_fd.sync()
|
||||
|
||||
# The on-line tracking works on a per-object basis...
|
||||
|
|
@ -267,7 +267,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
|
|||
self.assert_equal(len(self.repository), 3)
|
||||
|
||||
def test_ignores_commit_tag_in_data(self):
|
||||
self.repository.put(b'0' * 32, LoggedIO.COMMIT)
|
||||
self.repository.put(H(0), LoggedIO.COMMIT)
|
||||
self.reopen()
|
||||
with self.repository:
|
||||
io = self.repository.io
|
||||
|
|
@ -359,19 +359,19 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
|
|||
def test_append_only(self):
|
||||
def segments_in_repository():
|
||||
return len(list(self.repository.io.segment_iterator()))
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.commit()
|
||||
|
||||
self.repository.append_only = False
|
||||
assert segments_in_repository() == 2
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.commit()
|
||||
# normal: compact squashes the data together, only one segment
|
||||
assert segments_in_repository() == 4
|
||||
|
||||
self.repository.append_only = True
|
||||
assert segments_in_repository() == 4
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.commit()
|
||||
# append only: does not compact, only new segments written
|
||||
assert segments_in_repository() == 6
|
||||
|
|
@ -385,7 +385,7 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
|
|||
self.reopen()
|
||||
|
||||
with self.repository:
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foobar')
|
||||
self.repository.put(H(0), b'foobar')
|
||||
with pytest.raises(Repository.InsufficientFreeSpaceError):
|
||||
self.repository.commit()
|
||||
|
||||
|
|
@ -393,13 +393,13 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
|
|||
class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.repository.put(b'00000000000000000000000000000000', b'foo')
|
||||
self.repository.put(H(0), b'foo')
|
||||
self.repository.commit()
|
||||
self.repository.close()
|
||||
|
||||
def do_commit(self):
|
||||
with self.repository:
|
||||
self.repository.put(b'00000000000000000000000000000000', b'fox')
|
||||
self.repository.put(H(0), b'fox')
|
||||
self.repository.commit()
|
||||
|
||||
def test_corrupted_hints(self):
|
||||
|
|
@ -453,12 +453,12 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
|||
|
||||
def get_objects(self, *ids):
|
||||
for id_ in ids:
|
||||
self.repository.get(('%032d' % id_).encode('ascii'))
|
||||
self.repository.get(H(id_))
|
||||
|
||||
def add_objects(self, segments):
|
||||
for ids in segments:
|
||||
for id_ in ids:
|
||||
self.repository.put(('%032d' % id_).encode('ascii'), b'data')
|
||||
self.repository.put(H(id_), b'data')
|
||||
self.repository.commit()
|
||||
|
||||
def get_head(self):
|
||||
|
|
@ -469,7 +469,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
|||
|
||||
def corrupt_object(self, id_):
|
||||
idx = self.open_index()
|
||||
segment, offset = idx[('%032d' % id_).encode('ascii')]
|
||||
segment, offset = idx[H(id_)]
|
||||
with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd:
|
||||
fd.seek(offset)
|
||||
fd.write(b'BOOM')
|
||||
|
|
@ -560,8 +560,8 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
|||
self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects())
|
||||
|
||||
def test_crash_before_compact(self):
|
||||
self.repository.put(bytes(32), b'data')
|
||||
self.repository.put(bytes(32), b'data2')
|
||||
self.repository.put(H(0), b'data')
|
||||
self.repository.put(H(0), b'data2')
|
||||
# Simulate a crash before compact
|
||||
with patch.object(Repository, 'compact_segments') as compact:
|
||||
self.repository.commit()
|
||||
|
|
@ -569,7 +569,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
|
|||
self.reopen()
|
||||
with self.repository:
|
||||
self.check(repair=True)
|
||||
self.assert_equal(self.repository.get(bytes(32)), b'data2')
|
||||
self.assert_equal(self.repository.get(H(0)), b'data2')
|
||||
|
||||
|
||||
class RemoteRepositoryTestCase(RepositoryTestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue