Merge pull request #5057 from MichaelEischer/fix-backup-irregular

backup: fix handling of files with type irregular

changelog/unreleased/pull-5057

@@ -0,0 +1,21 @@
+Bugfix: Do not include irregular files in backup
+
+Since restic 0.17.1, files with type `irregular` could incorrectly be included
+in snapshots. This is most likely to occur when backing up special file types
+on Windows that cannot be handled by restic.
+
+This has been fixed.
+
+When running the `check` command this bug resulted in an error like the
+following:
+
+```
+  tree 12345678[...]: node "example.zip" with invalid type "irregular"
+```
+
+Repairing the affected snapshots requires upgrading to restic 0.17.2 and then
+manually running `restic repair snapshots --forget`. This will remove the
+`irregular` files from the snapshots.
+
+https://github.com/restic/restic/pull/5057
+https://forum.restic.net/t/errors-found-by-check-1-invalid-type-irregular-2-ciphertext-verification-failed/8447/2

cmd/restic/cmd_repair_snapshots.go

@@ -92,6 +92,10 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 	// - files whose contents are not fully available  (-> file will be modified)
 	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
 		RewriteNode: func(node *restic.Node, path string) *restic.Node {
+			if node.Type == restic.NodeTypeIrregular || node.Type == restic.NodeTypeInvalid {
+				Verbosef("  file %q: removed node with invalid type %q\n", path, node.Type)
+				return nil
+			}
 			if node.Type != restic.NodeTypeFile {
 				return node
 			}

internal/archiver/archiver.go

@@ -270,7 +270,8 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo,
 	}
 	// overwrite name to match that within the snapshot
 	node.Name = path.Base(snPath)
-	if err != nil {
+	// do not filter error for nodes of irregular or invalid type
+	if node.Type != restic.NodeTypeIrregular && node.Type != restic.NodeTypeInvalid && err != nil {
 		err = fmt.Errorf("incomplete metadata for %v: %w", filename, err)
 		return node, arch.error(filename, err)
 	}

internal/archiver/archiver_test.go

@@ -2407,4 +2407,47 @@ func TestMetadataBackupErrorFiltering(t *testing.T) {
 	rtest.Assert(t, node != nil, "node is missing")
 	rtest.Assert(t, err == replacementErr, "expected %v got %v", replacementErr, err)
 	rtest.Assert(t, filteredErr != nil, "missing inner error")
+
+	// check that errors from reading irregular file are not filtered
+	filteredErr = nil
+	node, err = arch.nodeFromFileInfo("file", filename, wrapIrregularFileInfo(fi), false)
+	rtest.Assert(t, node != nil, "node is missing")
+	rtest.Assert(t, filteredErr == nil, "error for irregular node should not have been filtered")
+	rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err)
+}
+
+func TestIrregularFile(t *testing.T) {
+	files := TestDir{
+		"testfile": TestFile{
+			Content: "foo bar test file",
+		},
+	}
+	tempdir, repo := prepareTempdirRepoSrc(t, files)
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	tempfile := filepath.Join(tempdir, "testfile")
+	fi := lstat(t, "testfile")
+
+	statfs := &StatFS{
+		FS: fs.Local{},
+		OverrideLstat: map[string]os.FileInfo{
+			tempfile: wrapIrregularFileInfo(fi),
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	arch := New(repo, fs.Track{FS: statfs}, Options{})
+	_, excluded, err := arch.save(ctx, "/", tempfile, nil)
+	if err == nil {
+		t.Fatalf("Save() should have failed")
+	}
+	rtest.Assert(t, strings.Contains(err.Error(), "irregular"), "unexpected error %q does not warn about irregular file mode", err)
+
+	if excluded {
+		t.Errorf("Save() excluded the node, that's unexpected")
+	}
 }

internal/archiver/archiver_unix_test.go

@@ -46,6 +46,16 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo {
 	return res
 }
 
+// wrapIrregularFileInfo returns a new os.FileInfo with the mode changed to irregular file
+func wrapIrregularFileInfo(fi os.FileInfo) os.FileInfo {
+	// wrap the os.FileInfo so we can return a modified stat_t
+	return wrappedFileInfo{
+		FileInfo: fi,
+		sys:      fi.Sys().(*syscall.Stat_t),
+		mode:     (fi.Mode() &^ os.ModeType) | os.ModeIrregular,
+	}
+}
+
 func statAndSnapshot(t *testing.T, repo archiverRepo, name string) (*restic.Node, *restic.Node) {
 	fi := lstat(t, name)
 	want, err := fs.NodeFromFileInfo(name, fi, false)

internal/archiver/archiver_windows_test.go

@@ -26,3 +26,11 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo {
 
 	return res
 }
+
+// wrapIrregularFileInfo returns a new os.FileInfo with the mode changed to irregular file
+func wrapIrregularFileInfo(fi os.FileInfo) os.FileInfo {
+	return wrappedFileInfo{
+		FileInfo: fi,
+		mode:     (fi.Mode() &^ os.ModeType) | os.ModeIrregular,
+	}
+}