From 90ca04f535fef019023cdb2c20a9a987515ce296 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 22 Aug 2022 11:35:44 +0200 Subject: [PATCH] repository api: flags support, fixes #6982 - .list: only return IDs for objects where flags & mask == value. - .flags(_many) (new) to set/query flags --- src/borg/remote.py | 18 ++++++++++-- src/borg/repository.py | 22 +++++++++++++-- src/borg/testsuite/repository.py | 48 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/borg/remote.py b/src/borg/remote.py index 016fda757..962f785fd 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -146,6 +146,8 @@ class RepositoryServer: # pragma: no cover "commit", "delete", "destroy", + "flags", + "flags_many", "get", "list", "scan", @@ -979,14 +981,26 @@ def destroy(self): def __len__(self): """actual remoting is done via self.call in the @api decorator""" - @api(since=parse_version("1.0.0")) - def list(self, limit=None, marker=None): + @api( + since=parse_version("1.0.0"), + mask={"since": parse_version("2.0.0b2"), "previously": 0}, + value={"since": parse_version("2.0.0b2"), "previously": 0}, + ) + def list(self, limit=None, marker=None, mask=0, value=0): """actual remoting is done via self.call in the @api decorator""" @api(since=parse_version("1.1.0b3")) def scan(self, limit=None, marker=None): """actual remoting is done via self.call in the @api decorator""" + @api(since=parse_version("2.0.0b2")) + def flags(self, id, mask=0xFFFFFFFF, value=None): + """actual remoting is done via self.call in the @api decorator""" + + @api(since=parse_version("2.0.0b2")) + def flags_many(self, ids, mask=0xFFFFFFFF, value=None): + """actual remoting is done via self.call in the @api decorator""" + def get(self, id): for resp in self.get_many([id]): return resp diff --git a/src/borg/repository.py b/src/borg/repository.py index 322dcfc03..6ed1a2689 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1197,13 +1197,15 @@ def __contains__(self, id): self.index = self.open_index(self.get_transaction_id()) return id in self.index - def list(self, limit=None, marker=None): + def list(self, limit=None, marker=None, mask=0, value=0): """ list IDs starting from after id - in index (pseudo-random) order. + + if mask and value are given, only return IDs where flags & mask == value (default: all IDs). """ if not self.index: self.index = self.open_index(self.get_transaction_id()) - return [id_ for id_, _ in islice(self.index.iteritems(marker=marker), limit)] + return [id_ for id_, _ in islice(self.index.iteritems(marker=marker, mask=mask, value=value), limit)] def scan(self, limit=None, marker=None): """ @@ -1250,6 +1252,22 @@ def scan(self, limit=None, marker=None): return result return result + def flags(self, id, mask=0xFFFFFFFF, value=None): + """ + query and optionally set flags + + :param id: id (key) of object + :param mask: bitmask for flags (default: operate on all 32 bits) + :param value: value to set masked bits to (default: do not change any flags) + :return: (previous) flags value (only masked bits) + """ + if not self.index: + self.index = self.open_index(self.get_transaction_id()) + return self.index.flags(id, mask, value) + + def flags_many(self, ids, mask=0xFFFFFFFF, value=None): + return [self.flags(id_, mask, value) for id_ in ids] + def get(self, id): if not self.index: self.index = self.open_index(self.get_transaction_id()) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 02d93e918..45669ff69 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -173,6 +173,54 @@ def test_max_data_size(self): self.assert_equal(self.repository.get(H(0)), max_data) self.assert_raises(IntegrityError, lambda: self.repository.put(H(1), max_data + b"x")) + def test_set_flags(self): + id = H(0) + self.repository.put(id, b"") + self.assert_equal(self.repository.flags(id), 0x00000000) # init == all zero + self.repository.flags(id, mask=0x00000001, value=0x00000001) + self.assert_equal(self.repository.flags(id), 0x00000001) + self.repository.flags(id, mask=0x00000002, value=0x00000002) + self.assert_equal(self.repository.flags(id), 0x00000003) + self.repository.flags(id, mask=0x00000001, value=0x00000000) + self.assert_equal(self.repository.flags(id), 0x00000002) + self.repository.flags(id, mask=0x00000002, value=0x00000000) + self.assert_equal(self.repository.flags(id), 0x00000000) + + def test_get_flags(self): + id = H(0) + self.repository.put(id, b"") + self.assert_equal(self.repository.flags(id), 0x00000000) # init == all zero + self.repository.flags(id, mask=0xC0000003, value=0x80000001) + self.assert_equal(self.repository.flags(id, mask=0x00000001), 0x00000001) + self.assert_equal(self.repository.flags(id, mask=0x00000002), 0x00000000) + self.assert_equal(self.repository.flags(id, mask=0x40000008), 0x00000000) + self.assert_equal(self.repository.flags(id, mask=0x80000000), 0x80000000) + + def test_flags_many(self): + ids_flagged = [H(0), H(1)] + ids_default_flags = [H(2), H(3)] + [self.repository.put(id, b"") for id in ids_flagged + ids_default_flags] + self.repository.flags_many(ids_flagged, mask=0xFFFFFFFF, value=0xDEADBEEF) + self.assert_equal(list(self.repository.flags_many(ids_default_flags)), [0x00000000, 0x00000000]) + self.assert_equal(list(self.repository.flags_many(ids_flagged)), [0xDEADBEEF, 0xDEADBEEF]) + self.assert_equal(list(self.repository.flags_many(ids_flagged, mask=0xFFFF0000)), [0xDEAD0000, 0xDEAD0000]) + self.assert_equal(list(self.repository.flags_many(ids_flagged, mask=0x0000FFFF)), [0x0000BEEF, 0x0000BEEF]) + + def test_flags_persistence(self): + self.repository.put(H(0), b"default") + self.repository.put(H(1), b"one one zero") + # we do not set flags for H(0), so we can later check their default state. + self.repository.flags(H(1), mask=0x00000007, value=0x00000006) + self.repository.commit(compact=False) + self.repository.close() + + self.repository = self.open() + with self.repository: + # we query all flags to check if the initial flags were all zero and + # only the ones we explicitly set to one are as expected. + self.assert_equal(self.repository.flags(H(0), mask=0xFFFFFFFF), 0x00000000) + self.assert_equal(self.repository.flags(H(1), mask=0xFFFFFFFF), 0x00000006) + class LocalRepositoryTestCase(RepositoryTestCaseBase): # test case that doesn't work with remote repositories