From ddb1c60964209117483e4bc6771b1c612aba5817 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 22 Aug 2016 04:42:42 +0200 Subject: [PATCH 1/3] repo tests: use H(x) instead of byte literals --- borg/testsuite/repository.py | 72 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 9c5a5a466..56e955785 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -61,59 +61,59 @@ class RepositoryTestCase(RepositoryTestCaseBase): def test2(self): """Test multiple sequential transactions """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'foo') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'foo') self.repository.commit() - self.repository.delete(b'00000000000000000000000000000000') - self.repository.put(b'00000000000000000000000000000001', b'bar') + self.repository.delete(H(0)) + self.repository.put(H(1), b'bar') self.repository.commit() - self.assert_equal(self.repository.get(b'00000000000000000000000000000001'), b'bar') + self.assert_equal(self.repository.get(H(1)), b'bar') def test_consistency(self): """Test cache consistency """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') - self.repository.put(b'00000000000000000000000000000000', b'foo2') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') - self.repository.put(b'00000000000000000000000000000000', b'bar') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'bar') - self.repository.delete(b'00000000000000000000000000000000') - self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(b'00000000000000000000000000000000')) + self.repository.put(H(0), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') + self.repository.put(H(0), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') + self.repository.put(H(0), b'bar') + self.assert_equal(self.repository.get(H(0)), b'bar') + self.repository.delete(H(0)) + self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(H(0))) def test_consistency2(self): """Test cache consistency2 """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') + self.repository.put(H(0), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') self.repository.commit() - self.repository.put(b'00000000000000000000000000000000', b'foo2') - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') + self.repository.put(H(0), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') self.repository.rollback() - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo') + self.assert_equal(self.repository.get(H(0)), b'foo') def test_overwrite_in_same_transaction(self): """Test cache consistency2 """ - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000000', b'foo2') + self.repository.put(H(0), b'foo') + self.repository.put(H(0), b'foo2') self.repository.commit() - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), b'foo2') + self.assert_equal(self.repository.get(H(0)), b'foo2') def test_single_kind_transactions(self): # put - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() self.repository.close() # replace self.repository = self.open() with self.repository: - self.repository.put(b'00000000000000000000000000000000', b'bar') + self.repository.put(H(0), b'bar') self.repository.commit() # delete self.repository = self.open() with self.repository: - self.repository.delete(b'00000000000000000000000000000000') + self.repository.delete(H(0)) self.repository.commit() def test_list(self): @@ -131,22 +131,22 @@ class RepositoryTestCase(RepositoryTestCaseBase): def test_max_data_size(self): max_data = b'x' * MAX_DATA_SIZE - self.repository.put(b'00000000000000000000000000000000', max_data) - self.assert_equal(self.repository.get(b'00000000000000000000000000000000'), max_data) + self.repository.put(H(0), max_data) + self.assert_equal(self.repository.get(H(0)), max_data) self.assert_raises(IntegrityError, - lambda: self.repository.put(b'00000000000000000000000000000001', max_data + b'x')) + lambda: self.repository.put(H(1), max_data + b'x')) class RepositoryCommitTestCase(RepositoryTestCaseBase): def add_keys(self): - self.repository.put(b'00000000000000000000000000000000', b'foo') - self.repository.put(b'00000000000000000000000000000001', b'bar') - self.repository.put(b'00000000000000000000000000000003', b'bar') + self.repository.put(H(0), b'foo') + self.repository.put(H(1), b'bar') + self.repository.put(H(3), b'bar') self.repository.commit() - self.repository.put(b'00000000000000000000000000000001', b'bar2') - self.repository.put(b'00000000000000000000000000000002', b'boo') - self.repository.delete(b'00000000000000000000000000000003') + self.repository.put(H(1), b'bar2') + self.repository.put(H(2), b'boo') + self.repository.delete(H(3)) def test_replay_of_missing_index(self): self.add_keys() @@ -264,19 +264,19 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase): def test_append_only(self): def segments_in_repository(): return len(list(self.repository.io.segment_iterator())) - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() self.repository.append_only = False assert segments_in_repository() == 1 - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() # normal: compact squashes the data together, only one segment assert segments_in_repository() == 1 self.repository.append_only = True assert segments_in_repository() == 1 - self.repository.put(b'00000000000000000000000000000000', b'foo') + self.repository.put(H(0), b'foo') self.repository.commit() # append only: does not compact, only new segments written assert segments_in_repository() == 2 From 0da0914955ee51aaa721617775f24813d01a4661 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 22 Aug 2016 04:52:18 +0200 Subject: [PATCH 2/3] repo tests: use H(x) instead of some similar constructs --- borg/testsuite/repository.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/borg/testsuite/repository.py b/borg/testsuite/repository.py index 56e955785..b95093e4a 100644 --- a/borg/testsuite/repository.py +++ b/borg/testsuite/repository.py @@ -44,8 +44,8 @@ class RepositoryTestCase(RepositoryTestCaseBase): def test1(self): for x in range(100): - self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') - key50 = ('%-32d' % 50).encode('ascii') + self.repository.put(H(x), b'SOMEDATA') + key50 = H(50) self.assert_equal(self.repository.get(key50), b'SOMEDATA') self.repository.delete(key50) self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50)) @@ -56,7 +56,7 @@ class RepositoryTestCase(RepositoryTestCaseBase): for x in range(100): if x == 50: continue - self.assert_equal(repository2.get(('%-32d' % x).encode('ascii')), b'SOMEDATA') + self.assert_equal(repository2.get(H(x)), b'SOMEDATA') def test2(self): """Test multiple sequential transactions @@ -118,7 +118,7 @@ class RepositoryTestCase(RepositoryTestCaseBase): def test_list(self): for x in range(100): - self.repository.put(('%-32d' % x).encode('ascii'), b'SOMEDATA') + self.repository.put(H(x), b'SOMEDATA') all = self.repository.list() self.assert_equal(len(all), 100) first_half = self.repository.list(limit=50) @@ -222,7 +222,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase): self.assert_equal(len(self.repository), 3) def test_ignores_commit_tag_in_data(self): - self.repository.put(b'0' * 32, LoggedIO.COMMIT) + self.repository.put(H(0), LoggedIO.COMMIT) self.reopen() with self.repository: io = self.repository.io @@ -294,12 +294,12 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): def get_objects(self, *ids): for id_ in ids: - self.repository.get(('%032d' % id_).encode('ascii')) + self.repository.get(H(id_)) def add_objects(self, segments): for ids in segments: for id_ in ids: - self.repository.put(('%032d' % id_).encode('ascii'), b'data') + self.repository.put(H(id_), b'data') self.repository.commit() def get_head(self): @@ -310,7 +310,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): def corrupt_object(self, id_): idx = self.open_index() - segment, offset = idx[('%032d' % id_).encode('ascii')] + segment, offset = idx[H(id_)] with open(os.path.join(self.tmppath, 'repository', 'data', '0', str(segment)), 'r+b') as fd: fd.seek(offset) fd.write(b'BOOM') @@ -401,8 +401,8 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.assert_equal(set([1, 2, 3, 4, 5, 6]), self.list_objects()) def test_crash_before_compact(self): - self.repository.put(bytes(32), b'data') - self.repository.put(bytes(32), b'data2') + self.repository.put(H(0), b'data') + self.repository.put(H(0), b'data2') # Simulate a crash before compact with patch.object(Repository, 'compact_segments') as compact: self.repository.commit() @@ -410,7 +410,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.reopen() with self.repository: self.check(repair=True) - self.assert_equal(self.repository.get(bytes(32)), b'data2') + self.assert_equal(self.repository.get(H(0)), b'data2') class RemoteRepositoryTestCase(RepositoryTestCase): From 93517ca30ef56f9f5e7d1caccf8d06e12242e64b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 22 Aug 2016 05:10:48 +0200 Subject: [PATCH 3/3] hashindex tests: use H(x) instead of some similar constructs note: hash values needed updating because H(x) formats differently. --- borg/testsuite/hashindex.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/borg/testsuite/hashindex.py b/borg/testsuite/hashindex.py index fba146952..75cd80227 100644 --- a/borg/testsuite/hashindex.py +++ b/borg/testsuite/hashindex.py @@ -23,19 +23,19 @@ class HashIndexTestCase(BaseTestCase): self.assert_equal(len(idx), 0) # Test set for x in range(100): - idx[bytes('%-32d' % x, 'ascii')] = make_value(x) + idx[H(x)] = make_value(x) self.assert_equal(len(idx), 100) for x in range(100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x)) + self.assert_equal(idx[H(x)], make_value(x)) # Test update for x in range(100): - idx[bytes('%-32d' % x, 'ascii')] = make_value(x * 2) + idx[H(x)] = make_value(x * 2) self.assert_equal(len(idx), 100) for x in range(100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) + self.assert_equal(idx[H(x)], make_value(x * 2)) # Test delete for x in range(50): - del idx[bytes('%-32d' % x, 'ascii')] + del idx[H(x)] self.assert_equal(len(idx), 50) idx_name = tempfile.NamedTemporaryFile() idx.write(idx_name.name) @@ -47,7 +47,7 @@ class HashIndexTestCase(BaseTestCase): idx = cls.read(idx_name.name) self.assert_equal(len(idx), 50) for x in range(50, 100): - self.assert_equal(idx[bytes('%-32d' % x, 'ascii')], make_value(x * 2)) + self.assert_equal(idx[H(x)], make_value(x * 2)) idx.clear() self.assert_equal(len(idx), 0) idx.write(idx_name.name) @@ -56,11 +56,11 @@ class HashIndexTestCase(BaseTestCase): def test_nsindex(self): self._generic_test(NSIndex, lambda x: (x, x), - '80fba5b40f8cf12f1486f1ba33c9d852fb2b41a5b5961d3b9d1228cf2aa9c4c9') + 'b96ec1ddabb4278cc92261ee171f7efc979dc19397cc5e89b778f05fa25bf93f') def test_chunkindex(self): self._generic_test(ChunkIndex, lambda x: (x, x, x), - '1d71865e72e3c3af18d3c7216b6fa7b014695eaa3ed7f14cf9cd02fba75d1c95') + '9d437a1e145beccc790c69e66ba94fc17bd982d83a401c9c6e524609405529d8') def test_resize(self): n = 2000 # Must be >= MIN_BUCKETS @@ -70,11 +70,11 @@ class HashIndexTestCase(BaseTestCase): initial_size = os.path.getsize(idx_name.name) self.assert_equal(len(idx), 0) for x in range(n): - idx[bytes('%-32d' % x, 'ascii')] = x, x + idx[H(x)] = x, x idx.write(idx_name.name) self.assert_true(initial_size < os.path.getsize(idx_name.name)) for x in range(n): - del idx[bytes('%-32d' % x, 'ascii')] + del idx[H(x)] self.assert_equal(len(idx), 0) idx.write(idx_name.name) self.assert_equal(initial_size, os.path.getsize(idx_name.name)) @@ -82,7 +82,7 @@ class HashIndexTestCase(BaseTestCase): def test_iteritems(self): idx = NSIndex() for x in range(100): - idx[bytes('%-0.32d' % x, 'ascii')] = x, x + idx[H(x)] = x, x all = list(idx.iteritems()) self.assert_equal(len(all), 100) second_half = list(idx.iteritems(marker=all[49][0]))