move include/exclude options to filter package

cmd/restic/cmd_backup.go

@@ -20,6 +20,7 @@ import (
 	"github.com/restic/restic/internal/archiver"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -66,7 +67,7 @@ Exit status is 12 if the password is incorrect.
 
 // BackupOptions bundles all options for the backup command.
 type BackupOptions struct {
-	excludePatternOptions
+	filter.ExcludePatternOptions
 
 	Parent            string
 	GroupBy           restic.SnapshotGroupByOptions
@@ -108,7 +109,7 @@ func init() {
 	f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
 	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
 
-	backupOptions.excludePatternOptions.Add(f)
+	backupOptions.ExcludePatternOptions.Add(f)
 
 	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
 	f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
@@ -297,7 +298,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 
 // collectRejectByNameFuncs returns a list of all functions which may reject data
 // from being saved in a snapshot based on path only
-func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
+func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
 	// exclude restic cache
 	if repo.Cache != nil {
 		f, err := rejectResticCache(repo)
@@ -308,11 +309,13 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
 		fs = append(fs, f)
 	}
 
-	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
+	fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
 	if err != nil {
 		return nil, err
 	}
-	fs = append(fs, fsPatterns...)
+	for _, pat := range fsPatterns {
+		fs = append(fs, archiver.RejectByNameFunc(pat))
+	}
 
 	return fs, nil
 }

cmd/restic/cmd_restore.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/restorer"
 	"github.com/restic/restic/internal/ui"
@@ -49,8 +50,8 @@ Exit status is 12 if the password is incorrect.
 
 // RestoreOptions collects all options for the restore command.
 type RestoreOptions struct {
-	excludePatternOptions
+	filter.ExcludePatternOptions
-	includePatternOptions
+	filter.IncludePatternOptions
 	Target string
 	restic.SnapshotFilter
 	DryRun    bool
@@ -68,8 +69,8 @@ func init() {
 	flags := cmdRestore.Flags()
 	flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
 
-	restoreOptions.excludePatternOptions.Add(flags)
+	restoreOptions.ExcludePatternOptions.Add(flags)
-	restoreOptions.includePatternOptions.Add(flags)
+	restoreOptions.IncludePatternOptions.Add(flags)
 
 	initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
 	flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done")
@@ -82,12 +83,12 @@ func init() {
 func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 	term *termstatus.Terminal, args []string) error {
 
-	excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
+	excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
 	if err != nil {
 		return err
 	}
 
-	includePatternFns, err := opts.includePatternOptions.CollectPatterns()
+	includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_rewrite.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
@@ -88,7 +89,7 @@ type RewriteOptions struct {
 
 	Metadata snapshotMetadataArgs
 	restic.SnapshotFilter
-	excludePatternOptions
+	filter.ExcludePatternOptions
 }
 
 var rewriteOptions RewriteOptions
@@ -103,7 +104,7 @@ func init() {
 	f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
 
 	initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
-	rewriteOptions.excludePatternOptions.Add(f)
+	rewriteOptions.ExcludePatternOptions.Add(f)
 }
 
 type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
@@ -113,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
 
-	rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
+	rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
 	if err != nil {
 		return false, err
 	}
@@ -263,7 +264,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
+	if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
 		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 

cmd/restic/cmd_rewrite_integration_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 	"github.com/restic/restic/internal/ui"
@@ -12,7 +13,7 @@ import (
 
 func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
 	opts := RewriteOptions{
-		excludePatternOptions: excludePatternOptions{
+		ExcludePatternOptions: filter.ExcludePatternOptions{
 			Excludes: excludes,
 		},
 		Forget:   forget,

cmd/restic/exclude.go

@@ -1,60 +1,16 @@
 package main
 
 import (
-	"bufio"
+	"github.com/restic/restic/internal/archiver"
-	"bytes"
-	"fmt"
-	"os"
-	"strings"
-
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/textfile"
-	"github.com/spf13/pflag"
 )
 
-// RejectByNameFunc is a function that takes a filename of a
-// file that would be included in the backup. The function returns true if it
-// should be excluded (rejected) from the backup.
-type RejectByNameFunc func(path string) bool
-
-// rejectByPattern returns a RejectByNameFunc which rejects files that match
-// one of the patterns.
-func rejectByPattern(patterns []string) RejectByNameFunc {
-	parsedPatterns := filter.ParsePatterns(patterns)
-	return func(item string) bool {
-		matched, err := filter.List(parsedPatterns, item)
-		if err != nil {
-			Warnf("error for exclude pattern: %v", err)
-		}
-
-		if matched {
-			debug.Log("path %q excluded by an exclude pattern", item)
-			return true
-		}
-
-		return false
-	}
-}
-
-// Same as `rejectByPattern` but case insensitive.
-func rejectByInsensitivePattern(patterns []string) RejectByNameFunc {
-	for index, path := range patterns {
-		patterns[index] = strings.ToLower(path)
-	}
-
-	rejFunc := rejectByPattern(patterns)
-	return func(item string) bool {
-		return rejFunc(strings.ToLower(item))
-	}
-}
-
 // rejectResticCache returns a RejectByNameFunc that rejects the restic cache
 // directory (if set).
-func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
+func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) {
 	if repo.Cache == nil {
 		return func(string) bool {
 			return false
@@ -75,115 +31,3 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
 		return false
 	}, nil
 }
-
-// readPatternsFromFiles reads all files and returns the list of
-// patterns. For each line, leading and trailing white space is removed
-// and comment lines are ignored. For each remaining pattern, environment
-// variables are resolved. For adding a literal dollar sign ($), write $$ to
-// the file.
-func readPatternsFromFiles(files []string) ([]string, error) {
-	getenvOrDollar := func(s string) string {
-		if s == "$" {
-			return "$"
-		}
-		return os.Getenv(s)
-	}
-
-	var patterns []string
-	for _, filename := range files {
-		err := func() (err error) {
-			data, err := textfile.Read(filename)
-			if err != nil {
-				return err
-			}
-
-			scanner := bufio.NewScanner(bytes.NewReader(data))
-			for scanner.Scan() {
-				line := strings.TrimSpace(scanner.Text())
-
-				// ignore empty lines
-				if line == "" {
-					continue
-				}
-
-				// strip comments
-				if strings.HasPrefix(line, "#") {
-					continue
-				}
-
-				line = os.Expand(line, getenvOrDollar)
-				patterns = append(patterns, line)
-			}
-			return scanner.Err()
-		}()
-		if err != nil {
-			return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
-		}
-	}
-	return patterns, nil
-}
-
-type excludePatternOptions struct {
-	Excludes                []string
-	InsensitiveExcludes     []string
-	ExcludeFiles            []string
-	InsensitiveExcludeFiles []string
-}
-
-func (opts *excludePatternOptions) Add(f *pflag.FlagSet) {
-	f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
-	f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
-}
-
-func (opts *excludePatternOptions) Empty() bool {
-	return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
-}
-
-func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
-	var fs []RejectByNameFunc
-	// add patterns from file
-	if len(opts.ExcludeFiles) > 0 {
-		excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if err := filter.ValidatePatterns(excludePatterns); err != nil {
-			return nil, errors.Fatalf("--exclude-file: %s", err)
-		}
-
-		opts.Excludes = append(opts.Excludes, excludePatterns...)
-	}
-
-	if len(opts.InsensitiveExcludeFiles) > 0 {
-		excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-
-		if err := filter.ValidatePatterns(excludes); err != nil {
-			return nil, errors.Fatalf("--iexclude-file: %s", err)
-		}
-
-		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludes) > 0 {
-		if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
-			return nil, errors.Fatalf("--iexclude: %s", err)
-		}
-
-		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
-	}
-
-	if len(opts.Excludes) > 0 {
-		if err := filter.ValidatePatterns(opts.Excludes); err != nil {
-			return nil, errors.Fatalf("--exclude: %s", err)
-		}
-
-		fs = append(fs, rejectByPattern(opts.Excludes))
-	}
-	return fs, nil
-}

cmd/restic/integration_filter_pattern_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/restic/restic/internal/filter"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -17,14 +18,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
 	var err error
 
 	// Test --exclude
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iexclude
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
@@ -47,14 +48,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
 	var err error
 
 	// Test --exclude-file:
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iexclude-file
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
+	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
@@ -70,28 +71,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
 	var err error
 
 	// Test --exclude
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iexclude
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --include
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
 	// Test --iinclude
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
 
 	rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
 *[._]log[.-][0-9]
@@ -111,22 +112,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
 		t.Fatalf("Could not write include file: %v", fileErr)
 	}
 
-	err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
+	err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())
 
-	err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
+	err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
 	rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
 *[._]log[.-][0-9]
 !*[._]log[.-][0-9]`, err.Error())

internal/archiver/exclude.go

@@ -14,6 +14,16 @@ import (
 	"github.com/restic/restic/internal/ui"
 )
 
+// RejectByNameFunc is a function that takes a filename of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectByNameFunc func(path string) bool
+
+// RejectFunc is a function that takes a filename and os.FileInfo of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
+
 type rejectionCache struct {
 	m   map[string]bool
 	mtx sync.Mutex
@@ -49,11 +59,6 @@ func (rc *rejectionCache) Store(dir string, rejected bool) {
 	rc.m[dir] = rejected
 }
 
-// RejectFunc is a function that takes a filename and os.FileInfo of a
-// file that would be included in the backup. The function returns true if it
-// should be excluded (rejected) from the backup.
-type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
-
 // RejectIfPresent returns a RejectByNameFunc which itself returns whether a path
 // should be excluded. The RejectByNameFunc considers a file to be excluded when
 // it resides in a directory with an exclusion file, that is specified by

internal/filter/exclude.go

@@ -0,0 +1,162 @@
+package filter
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/textfile"
+	"github.com/spf13/pflag"
+)
+
+// RejectByNameFunc is a function that takes a filename of a
+// file that would be included in the backup. The function returns true if it
+// should be excluded (rejected) from the backup.
+type RejectByNameFunc func(path string) bool
+
+// RejectByPattern returns a RejectByNameFunc which rejects files that match
+// one of the patterns.
+func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
+	parsedPatterns := ParsePatterns(patterns)
+	return func(item string) bool {
+		matched, err := List(parsedPatterns, item)
+		if err != nil {
+			warnf("error for exclude pattern: %v", err)
+		}
+
+		if matched {
+			debug.Log("path %q excluded by an exclude pattern", item)
+			return true
+		}
+
+		return false
+	}
+}
+
+// RejectByInsensitivePattern is like RejectByPattern but case insensitive.
+func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
+	for index, path := range patterns {
+		patterns[index] = strings.ToLower(path)
+	}
+
+	rejFunc := RejectByPattern(patterns, warnf)
+	return func(item string) bool {
+		return rejFunc(strings.ToLower(item))
+	}
+}
+
+// readPatternsFromFiles reads all files and returns the list of
+// patterns. For each line, leading and trailing white space is removed
+// and comment lines are ignored. For each remaining pattern, environment
+// variables are resolved. For adding a literal dollar sign ($), write $$ to
+// the file.
+func readPatternsFromFiles(files []string) ([]string, error) {
+	getenvOrDollar := func(s string) string {
+		if s == "$" {
+			return "$"
+		}
+		return os.Getenv(s)
+	}
+
+	var patterns []string
+	for _, filename := range files {
+		err := func() (err error) {
+			data, err := textfile.Read(filename)
+			if err != nil {
+				return err
+			}
+
+			scanner := bufio.NewScanner(bytes.NewReader(data))
+			for scanner.Scan() {
+				line := strings.TrimSpace(scanner.Text())
+
+				// ignore empty lines
+				if line == "" {
+					continue
+				}
+
+				// strip comments
+				if strings.HasPrefix(line, "#") {
+					continue
+				}
+
+				line = os.Expand(line, getenvOrDollar)
+				patterns = append(patterns, line)
+			}
+			return scanner.Err()
+		}()
+		if err != nil {
+			return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
+		}
+	}
+	return patterns, nil
+}
+
+type ExcludePatternOptions struct {
+	Excludes                []string
+	InsensitiveExcludes     []string
+	ExcludeFiles            []string
+	InsensitiveExcludeFiles []string
+}
+
+func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) {
+	f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
+	f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
+	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+}
+
+func (opts *ExcludePatternOptions) Empty() bool {
+	return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
+}
+
+func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) {
+	var fs []RejectByNameFunc
+	// add patterns from file
+	if len(opts.ExcludeFiles) > 0 {
+		excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := ValidatePatterns(excludePatterns); err != nil {
+			return nil, errors.Fatalf("--exclude-file: %s", err)
+		}
+
+		opts.Excludes = append(opts.Excludes, excludePatterns...)
+	}
+
+	if len(opts.InsensitiveExcludeFiles) > 0 {
+		excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := ValidatePatterns(excludes); err != nil {
+			return nil, errors.Fatalf("--iexclude-file: %s", err)
+		}
+
+		opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
+	}
+
+	if len(opts.InsensitiveExcludes) > 0 {
+		if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil {
+			return nil, errors.Fatalf("--iexclude: %s", err)
+		}
+
+		fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf))
+	}
+
+	if len(opts.Excludes) > 0 {
+		if err := ValidatePatterns(opts.Excludes); err != nil {
+			return nil, errors.Fatalf("--exclude: %s", err)
+		}
+
+		fs = append(fs, RejectByPattern(opts.Excludes, warnf))
+	}
+	return fs, nil
+}

internal/filter/exclude_test.go

@@ -1,4 +1,4 @@
-package main
+package filter
 
 import (
 	"testing"
@@ -21,7 +21,7 @@ func TestRejectByPattern(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run("", func(t *testing.T) {
-			reject := rejectByPattern(patterns)
+			reject := RejectByPattern(patterns, nil)
 			res := reject(tc.filename)
 			if res != tc.reject {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",
@@ -48,7 +48,7 @@ func TestRejectByInsensitivePattern(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run("", func(t *testing.T) {
-			reject := rejectByInsensitivePattern(patterns)
+			reject := RejectByInsensitivePattern(patterns, nil)
 			res := reject(tc.filename)
 			if res != tc.reject {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",

internal/filter/include.go

@@ -1,10 +1,9 @@
-package main
+package filter
 
 import (
 	"strings"
 
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/filter"
 	"github.com/spf13/pflag"
 )
 
@@ -12,21 +11,21 @@ import (
 // in the restore process and returns whether it should be included.
 type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
 
-type includePatternOptions struct {
+type IncludePatternOptions struct {
 	Includes                []string
 	InsensitiveIncludes     []string
 	IncludeFiles            []string
 	InsensitiveIncludeFiles []string
 }
 
-func (opts *includePatternOptions) Add(f *pflag.FlagSet) {
+func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
 	f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
 	f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
 }
 
-func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {
+func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
 	var fs []IncludeByNameFunc
 	if len(opts.IncludeFiles) > 0 {
 		includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
@@ -34,7 +33,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
 			return nil, err
 		}
 
-		if err := filter.ValidatePatterns(includePatterns); err != nil {
+		if err := ValidatePatterns(includePatterns); err != nil {
 			return nil, errors.Fatalf("--include-file: %s", err)
 		}
 
@@ -47,7 +46,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
 			return nil, err
 		}
 
-		if err := filter.ValidatePatterns(includePatterns); err != nil {
+		if err := ValidatePatterns(includePatterns); err != nil {
 			return nil, errors.Fatalf("--iinclude-file: %s", err)
 		}
 
@@ -55,45 +54,45 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
 	}
 
 	if len(opts.InsensitiveIncludes) > 0 {
-		if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil {
+		if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil {
 			return nil, errors.Fatalf("--iinclude: %s", err)
 		}
 
-		fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes))
+		fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf))
 	}
 
 	if len(opts.Includes) > 0 {
-		if err := filter.ValidatePatterns(opts.Includes); err != nil {
+		if err := ValidatePatterns(opts.Includes); err != nil {
 			return nil, errors.Fatalf("--include: %s", err)
 		}
 
-		fs = append(fs, includeByPattern(opts.Includes))
+		fs = append(fs, IncludeByPattern(opts.Includes, warnf))
 	}
 	return fs, nil
 }
 
-// includeByPattern returns a IncludeByNameFunc which includes files that match
+// IncludeByPattern returns a IncludeByNameFunc which includes files that match
 // one of the patterns.
-func includeByPattern(patterns []string) IncludeByNameFunc {
+func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
-	parsedPatterns := filter.ParsePatterns(patterns)
+	parsedPatterns := ParsePatterns(patterns)
 	return func(item string) (matched bool, childMayMatch bool) {
-		matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item)
+		matched, childMayMatch, err := ListWithChild(parsedPatterns, item)
 		if err != nil {
-			Warnf("error for include pattern: %v", err)
+			warnf("error for include pattern: %v", err)
 		}
 
 		return matched, childMayMatch
 	}
 }
 
-// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
+// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
 // one of the patterns, ignoring the casing of the filenames.
-func includeByInsensitivePattern(patterns []string) IncludeByNameFunc {
+func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
 	for index, path := range patterns {
 		patterns[index] = strings.ToLower(path)
 	}
 
-	includeFunc := includeByPattern(patterns)
+	includeFunc := IncludeByPattern(patterns, warnf)
 	return func(item string) (matched bool, childMayMatch bool) {
 		return includeFunc(strings.ToLower(item))
 	}

internal/filter/include_test.go

@@ -1,4 +1,4 @@
-package main
+package filter
 
 import (
 	"testing"
@@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.filename, func(t *testing.T) {
-			includeFunc := includeByPattern(patterns)
+			includeFunc := IncludeByPattern(patterns, nil)
 			matched, _ := includeFunc(tc.filename)
 			if matched != tc.include {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",
@@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.filename, func(t *testing.T) {
-			includeFunc := includeByInsensitivePattern(patterns)
+			includeFunc := IncludeByInsensitivePattern(patterns, nil)
 			matched, _ := includeFunc(tc.filename)
 			if matched != tc.include {
 				t.Fatalf("wrong result for filename %v: want %v, got %v",