backup: allow excluding online-only cloud files

changelog/unreleased/issue-3697

@@ -0,0 +1,11 @@
+Enhancement: Allow excluding online-only cloud files (such as OneDrive Files On-Demand)
+
+Restic treated OneDrive Files On-Demand as though they were regular files
+for the purpose of backup which caused issues with VSS, could make backup
+incredibly slow (as OneDrive attempted to download files), or could fill
+the source disk (e.g. 1TB of files in OneDrive on a 500GB disk).
+Restic now allows the user to exclude these files when backing up with
+the `--exclude-cloud-files` switch.
+
+https://github.com/restic/restic/issues/3697
+https://github.com/restic/restic/issues/4935

cmd/restic/cmd_backup.go

@@ -73,6 +73,7 @@ type BackupOptions struct {
 	ExcludeIfPresent  []string
 	ExcludeCaches     bool
 	ExcludeLargerThan string
+	ExcludeCloudFiles bool
 	Stdin             bool
 	StdinFilename     string
 	StdinCommand      bool
@@ -135,6 +136,7 @@ func init() {
 	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
 	if runtime.GOOS == "windows" {
 		f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
+		f.BoolVar(&backupOptions.ExcludeCloudFiles, "exclude-cloud-files", false, "excludes online-only cloud files (such as OneDrive Files On-Demand)")
 	}
 	f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
 
@@ -348,6 +350,14 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc,
 		fs = append(fs, f)
 	}
 
+	if opts.ExcludeCloudFiles && !opts.Stdin {
+		f, err := rejectCloudFiles()
+		if err != nil {
+			return nil, err
+		}
+		fs = append(fs, f)
+	}
+
 	return fs, nil
 }
 

cmd/restic/exclude.go

@@ -7,8 +7,10 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
+	"syscall"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
@@ -385,6 +387,33 @@ func rejectBySize(maxSizeStr string) (RejectFunc, error) {
 	}, nil
 }
 
+// rejectCloudFiles returns a func which rejects files which are online-only cloud files
+func rejectCloudFiles() (RejectFunc, error) {
+	if runtime.GOOS != "windows" {
+		return nil, errors.Fatalf("exclude-cloud-files is only supported on Windows")
+	}
+
+	const (
+		FILE_ATTRIBUTE_UNPINNED              = 0x00100000
+		FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS = 0x00400000
+	)
+
+	return func(item string, fi os.FileInfo) bool {
+		attrs, ok := fi.Sys().(*syscall.Win32FileAttributeData)
+		if !ok {
+			Warnf("could not determine file attributes: %s", item)
+			return false
+		}
+
+		if attrs.FileAttributes&(FILE_ATTRIBUTE_UNPINNED|FILE_ATTRIBUTE_RECALL_ON_DATA_ACCESS) > 0 {
+			debug.Log("rejecting online-only cloud file %s", item)
+			return true
+		}
+
+		return false
+	}, nil
+}
+
 // readPatternsFromFiles reads all files and returns the list of
 // patterns. For each line, leading and trailing white space is removed
 // and comment lines are ignored. For each remaining pattern, environment

doc/040_backup.rst

@@ -297,7 +297,8 @@ the exclude options are:
 -  ``--exclude-file`` Specified one or more times to exclude items listed in a given file
 -  ``--iexclude-file`` Same as ``exclude-file`` but ignores cases like in ``--iexclude``
 -  ``--exclude-if-present foo`` Specified one or more times to exclude a folder's content if it contains a file called ``foo`` (optionally having a given header, no wildcards for the file name supported)
--  ``--exclude-larger-than size`` Specified once to excludes files larger than the given size
+-  ``--exclude-larger-than size`` Specified once to exclude files larger than the given size
+-  ``--exclude-cloud-files`` Specified once to exclude online-only cloud files (such as OneDrive Files On-Demand), currently only supported on Windows
 
 Please see ``restic help backup`` for more specific information about each exclude option.