From 4f9cda1aab1b937b126fdd89d6a8c084a5c0abc8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Jan 2023 23:47:18 +0100 Subject: [PATCH 1/7] get_item_uid_gid: do not require item.uid/gid, see #7249 if uid is not present, fall back to uid_default. if gid is not present, fall back to gid_default. --- src/borg/archive.py | 8 ++++---- src/borg/testsuite/archive.py | 26 ++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 8a7564b6f..141022a3c 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -410,15 +410,15 @@ def get_item_uid_gid(item, *, numeric, uid_forced=None, gid_forced=None, uid_def uid = uid_forced else: uid = None if numeric else user2uid(item.get("user")) - uid = item.uid if uid is None else uid - if uid < 0: + uid = item.get("uid") if uid is None else uid + if uid is None or uid < 0: uid = uid_default if gid_forced is not None: gid = gid_forced else: gid = None if numeric else group2gid(item.get("group")) - gid = item.gid if gid is None else gid - if gid < 0: + gid = item.get("gid") if gid is None else gid + if gid is None or gid < 0: gid = gid_default return uid, gid diff --git a/src/borg/testsuite/archive.py b/src/borg/testsuite/archive.py index 3c518ffd3..2c67cd8db 100644 --- a/src/borg/testsuite/archive.py +++ b/src/borg/testsuite/archive.py @@ -356,3 +356,29 @@ def test_get_item_uid_gid(): # because item uid/gid seems valid, do not use the given uid/gid defaults assert uid == 9 assert gid == 10 + + # item metadata only has uid/gid, but no user/group. + item = Item(path="filename", uid=13, gid=14) + + uid, gid = get_item_uid_gid(item, numeric=False) + # it'll check user/group first, but as there is nothing in the item, falls back to uid/gid. + assert uid == 13 + assert gid == 14 + + uid, gid = get_item_uid_gid(item, numeric=True) + # does not check user/group, directly returns uid/gid. + assert uid == 13 + assert gid == 14 + + # item metadata has no uid/gid/user/group. + item = Item(path="filename") + + uid, gid = get_item_uid_gid(item, numeric=False, uid_default=15) + # as there is nothing, it'll fall back to uid_default/gid_default. + assert uid == 15 + assert gid == 0 + + uid, gid = get_item_uid_gid(item, numeric=True, gid_default=16) + # as there is nothing, it'll fall back to uid_default/gid_default. + assert uid == 0 + assert gid == 16 From 262812e76f7abc5a164ff8801e42b1dc5ce8be12 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 14 Jan 2023 23:54:08 +0100 Subject: [PATCH 2/7] get_item_data: do not require item.uid/gid, see #7249 if uid is not present, use None. if gid is not present, use None. --- src/borg/helpers/parseformat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index f1142871d..94d005e4f 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -893,10 +893,10 @@ def get_item_data(self, item): item_data["type"] = item_type item_data["mode"] = mode - item_data.update(text_to_json("user", item.get("user", str(item.uid)))) - item_data.update(text_to_json("group", item.get("group", str(item.gid)))) - item_data["uid"] = item.uid - item_data["gid"] = item.gid + item_data["uid"] = item.get("uid") # int or None + item_data["gid"] = item.get("gid") # int or None + item_data.update(text_to_json("user", item.get("user", str(item_data["uid"])))) + item_data.update(text_to_json("group", item.get("group", str(item_data["gid"])))) if self.json_lines: item_data["healthy"] = "chunks_healthy" not in item From b338eb0ce88dda50d882e7f05674be17a45119ca Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Jan 2023 00:05:50 +0100 Subject: [PATCH 3/7] process_pipe: allow creating item w/o user/group/uid/gid, see #7249 --- src/borg/archive.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 141022a3c..93e90b194 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -1399,28 +1399,32 @@ def process_symlink(self, *, path, parent_fd, name, st): item.update(self.metadata_collector.stat_attrs(st, path)) # can't use FD here? return status - def process_pipe(self, *, path, cache, fd, mode, user, group): + def process_pipe(self, *, path, cache, fd, mode, user=None, group=None): status = "i" # stdin (or other pipe) self.print_file_status(status, path) status = None # we already printed the status - uid = user2uid(user) - if uid is None: - raise Error("no such user: %s" % user) - gid = group2gid(group) - if gid is None: - raise Error("no such group: %s" % group) + if user is not None: + uid = user2uid(user) + if uid is None: + raise Error("no such user: %s" % user) + else: + uid = None + if group is not None: + gid = group2gid(group) + if gid is None: + raise Error("no such group: %s" % group) + else: + gid = None t = int(time.time()) * 1000000000 - item = Item( - path=path, - mode=mode & 0o107777 | 0o100000, # forcing regular file mode - uid=uid, - user=user, - gid=gid, - group=group, - mtime=t, - atime=t, - ctime=t, - ) + item = Item(path=path, mode=mode & 0o107777 | 0o100000, mtime=t, atime=t, ctime=t) # forcing regular file mode + if user is not None: + item.user = user + if group is not None: + item.group = group + if uid is not None: + item.uid = uid + if gid is not None: + item.gid = gid self.process_file_chunks(item, cache, self.stats, self.show_progress, backup_io_iter(self.chunker.chunkify(fd))) item.get_size(memorize=True) self.stats.nfiles += 1 From 1f9c46f2b56cbac7edab2bfe7e5d2d786d567aa6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Jan 2023 00:13:18 +0100 Subject: [PATCH 4/7] create: do not store user/group for stdin data by default, see #7249 if you want user/group stored, give --stdin-user=USER / --stdin-group=GROUP. --- src/borg/archiver/create_cmd.py | 9 ++++----- src/borg/testsuite/archiver/create_cmd.py | 4 ---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 0cb51fa42..4f8208ab5 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -30,7 +30,6 @@ from ..patterns import PatternMatcher from ..platform import is_win32 from ..platform import get_flags -from ..platform import uid2user, gid2group from ..logger import create_logger @@ -718,15 +717,15 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): "--stdin-user", metavar="USER", dest="stdin_user", - default=uid2user(0), - help="set user USER in archive for stdin data (default: %(default)r)", + default=None, + help="set user USER in archive for stdin data (default: do not store user/uid)", ) subparser.add_argument( "--stdin-group", metavar="GROUP", dest="stdin_group", - default=gid2group(0), - help="set group GROUP in archive for stdin data (default: %(default)r)", + default=None, + help="set group GROUP in archive for stdin data (default: do not store group/gid)", ) subparser.add_argument( "--stdin-mode", diff --git a/src/borg/testsuite/archiver/create_cmd.py b/src/borg/testsuite/archiver/create_cmd.py index 76e9a0209..8b9f1d4d1 100644 --- a/src/borg/testsuite/archiver/create_cmd.py +++ b/src/borg/testsuite/archiver/create_cmd.py @@ -160,8 +160,6 @@ def test_create_stdin(self): input_data = b"\x00foo\n\nbar\n \n" self.cmd(f"--repo={self.repository_location}", "create", "test", "-", input=input_data) item = json.loads(self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines")) - assert item["uid"] == 0 - assert item["gid"] == 0 assert item["size"] == len(input_data) assert item["path"] == "stdin" extracted_data = self.cmd( @@ -185,8 +183,6 @@ def test_create_content_from_command(self): input_data, ) item = json.loads(self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines")) - assert item["uid"] == 0 - assert item["gid"] == 0 assert item["size"] == len(input_data) + 1 # `echo` adds newline assert item["path"] == name extracted_data = self.cmd(f"--repo={self.repository_location}", "extract", "test", "--stdout") From 4dcc48f5c417bb323a85a8533f6d534e9eb664e8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Jan 2023 00:28:29 +0100 Subject: [PATCH 5/7] extract: chown only if we have u/g info in archived item, see #7249 also: move get_item_uid_gid() to "not is_win32" block for now. --- src/borg/archive.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index 93e90b194..6d8e4e04d 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -928,9 +928,12 @@ def restore_attrs(self, path, item, symlink=False, fd=None): Does not access the repository. """ backup_io.op = "attrs" - uid, gid = get_item_uid_gid(item, numeric=self.numeric_ids) - # This code is a bit of a mess due to os specific differences + # This code is a bit of a mess due to OS specific differences. if not is_win32: + # by using uid_default = -1 and gid_default = -1, they will not be restored if + # the archived item has no information about them. + uid, gid = get_item_uid_gid(item, numeric=self.numeric_ids, uid_default=-1, gid_default=-1) + # if uid and/or gid is -1, chown will keep it as is and not change it. try: if fd: os.fchown(fd, uid, gid) From ff545033e316c605f876f92075ea289256cd23aa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Jan 2023 01:38:13 +0100 Subject: [PATCH 6/7] tests: do not look up uid 0 / gid 0, but current process uid/gid some systems do not have uid/gid 0 (windows). --- src/borg/testsuite/archive.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/borg/testsuite/archive.py b/src/borg/testsuite/archive.py index 2c67cd8db..c796b6af3 100644 --- a/src/borg/testsuite/archive.py +++ b/src/borg/testsuite/archive.py @@ -1,4 +1,5 @@ import json +import os from collections import OrderedDict from datetime import datetime, timezone from io import StringIO @@ -295,18 +296,22 @@ def __next__(self): def test_get_item_uid_gid(): # test requires that: - # - a name for user 0 and group 0 exists, usually root:root or root:wheel. + # - a user/group name for the current process' real uid/gid exists. # - a system user/group udoesnotexist:gdoesnotexist does NOT exist. - user0, group0 = uid2user(0), gid2group(0) + try: + puid, pgid = os.getuid(), os.getgid() # UNIX only + except AttributeError: + puid, pgid = 0, 0 + puser, pgroup = uid2user(puid), gid2group(pgid) # this is intentionally a "strange" item, with not matching ids/names. - item = Item(path="filename", uid=1, gid=2, user=user0, group=group0) + item = Item(path="filename", uid=1, gid=2, user=puser, group=pgroup) uid, gid = get_item_uid_gid(item, numeric=False) # these are found via a name-to-id lookup - assert uid == 0 - assert gid == 0 + assert uid == puid + assert gid == pgid uid, gid = get_item_uid_gid(item, numeric=True) # these are directly taken from the item.uid and .gid @@ -319,7 +324,7 @@ def test_get_item_uid_gid(): assert gid == 4 # item metadata broken, has negative ids. - item = Item(path="filename", uid=-1, gid=-2, user=user0, group=group0) + item = Item(path="filename", uid=-1, gid=-2, user=puser, group=pgroup) uid, gid = get_item_uid_gid(item, numeric=True) # use the uid/gid defaults (which both default to 0). From 18898d68f008e98856ec5a131fa3b5f74a30a22c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Jan 2023 01:57:19 +0100 Subject: [PATCH 7/7] export-tar: for items w/o uid/gid, default to 0/0, see #7249 --- src/borg/archiver/tar_cmds.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 2ee5a0712..c273ffde7 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -132,8 +132,8 @@ def item_to_tarinfo(item, original_path): tarinfo.name = item.path tarinfo.mtime = item.mtime / 1e9 tarinfo.mode = stat.S_IMODE(item.mode) - tarinfo.uid = item.uid - tarinfo.gid = item.gid + tarinfo.uid = item.get("uid", 0) + tarinfo.gid = item.get("gid", 0) tarinfo.uname = item.get("user", "") tarinfo.gname = item.get("group", "") # The linkname in tar has 2 uses: