diff --git a/src/borg/archive.py b/src/borg/archive.py
index e3fcbe5eb..75aefae80 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -920,11 +920,7 @@ def restore_attrs(self, path, item, symlink=False, fd=None):
                     if not symlink:
                         os.chmod(path, item.mode)
             mtime = item.mtime
-            if "atime" in item:
-                atime = item.atime
-            else:
-                # old archives only had mtime in item metadata
-                atime = mtime
+            atime = item.atime if "atime" in item else mtime
             if "birthtime" in item:
                 birthtime = item.birthtime
                 try:
@@ -1119,10 +1115,11 @@ def __init__(self, *, noatime, noctime, nobirthtime, numeric_ids, noflags, noacl
         self.nobirthtime = nobirthtime
 
     def stat_simple_attrs(self, st):
-        attrs = dict(mode=st.st_mode, uid=st.st_uid, gid=st.st_gid, mtime=safe_ns(st.st_mtime_ns))
+        attrs = dict(mode=st.st_mode, uid=st.st_uid, gid=st.st_gid)
         # borg can work with archives only having mtime (very old borg archives do not have
         # atime/ctime). it can be useful to omit atime/ctime, if they change without the
         # file content changing - e.g. to get better metadata deduplication.
+        attrs["mtime"] = safe_ns(st.st_mtime_ns)
         if not self.noatime:
             attrs["atime"] = safe_ns(st.st_atime_ns)
         if not self.noctime: