diff --git a/changelog/unreleased/issue-5003 b/changelog/unreleased/issue-5003
new file mode 100644
index 000000000..d02b06bc7
--- /dev/null
+++ b/changelog/unreleased/issue-5003
@@ -0,0 +1,14 @@
+Bugfix: fix metadata errors during backup of removable disks on Windows
+
+Since restic 0.17.0, backups of removable disks on Windows could report
+errors with retrieving metadata like shown below.
+
+```
+error: incomplete metadata for d:\filename: get named security info failed with: Access is denied.
+```
+
+This has now been fixed.
+
+https://github.com/restic/restic/issues/5003
+https://github.com/restic/restic/pull/5123
+https://forum.restic.net/t/backing-up-a-folder-from-a-veracrypt-volume-brings-up-errors-since-restic-v17-0/8444
diff --git a/internal/fs/sd_windows.go b/internal/fs/sd_windows.go
index 66d9bcb54..6bffa4fe2 100644
--- a/internal/fs/sd_windows.go
+++ b/internal/fs/sd_windows.go
@@ -54,6 +54,15 @@ func getSecurityDescriptor(filePath string) (securityDescriptor *[]byte, err err
 		sd, err = getNamedSecurityInfoLow(filePath)
 	} else {
 		sd, err = getNamedSecurityInfoHigh(filePath)
+		// Fallback to the low privilege version when receiving an access denied error.
+		// For some reason the ERROR_PRIVILEGE_NOT_HELD error is not returned for removable media
+		// but instead an access denied error is returned. Workaround that by just retrying with
+		// the low privilege version, but don't switch privileges as we cannot distinguish this
+		// case from actual access denied errors.
+		// see https://github.com/restic/restic/issues/5003#issuecomment-2452314191 for details
+		if err != nil && isAccessDeniedError(err) {
+			sd, err = getNamedSecurityInfoLow(filePath)
+		}
 	}
 	if err != nil {
 		if !useLowerPrivileges && isHandlePrivilegeNotHeldError(err) {
@@ -114,6 +123,10 @@ func setSecurityDescriptor(filePath string, securityDescriptor *[]byte) error {
 		err = setNamedSecurityInfoLow(filePath, dacl)
 	} else {
 		err = setNamedSecurityInfoHigh(filePath, owner, group, dacl, sacl)
+		// See corresponding fallback in getSecurityDescriptor for an explanation
+		if err != nil && isAccessDeniedError(err) {
+			err = setNamedSecurityInfoLow(filePath, dacl)
+		}
 	}
 
 	if err != nil {
@@ -174,6 +187,15 @@ func isHandlePrivilegeNotHeldError(err error) bool {
 	return false
 }
 
+// isAccessDeniedError checks if the error is ERROR_ACCESS_DENIED
+func isAccessDeniedError(err error) bool {
+	if errno, ok := err.(syscall.Errno); ok {
+		// Compare the error code to the expected value
+		return errno == windows.ERROR_ACCESS_DENIED
+	}
+	return false
+}
+
 // securityDescriptorBytesToStruct converts the security descriptor bytes representation
 // into a pointer to windows SECURITY_DESCRIPTOR.
 func securityDescriptorBytesToStruct(sd []byte) (*windows.SECURITY_DESCRIPTOR, error) {