mirror of
https://github.com/restic/restic.git
synced 2024-12-21 23:33:03 +00:00
move include/exclude options to filter package
This commit is contained in:
Michael Eischer
parent
5d58945718
commit
f1585af0f2
11 changed files with 233 additions and 216 deletions
|
|
@ -20,6 +20,7 @@ import (
|
|||
"github.com/restic/restic/internal/archiver"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
|
|
@ -66,7 +67,7 @@ Exit status is 12 if the password is incorrect.
|
|||
|
||||
// BackupOptions bundles all options for the backup command.
|
||||
type BackupOptions struct {
|
||||
excludePatternOptions
|
||||
filter.ExcludePatternOptions
|
||||
|
||||
Parent string
|
||||
GroupBy restic.SnapshotGroupByOptions
|
||||
|
|
@ -108,7 +109,7 @@ func init() {
|
|||
f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
|
||||
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
|
||||
|
||||
backupOptions.excludePatternOptions.Add(f)
|
||||
backupOptions.ExcludePatternOptions.Add(f)
|
||||
|
||||
f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems, don't cross filesystem boundaries and subvolumes")
|
||||
f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes `filename[:header]`, exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)")
|
||||
|
|
@ -297,7 +298,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
|||
|
||||
// collectRejectByNameFuncs returns a list of all functions which may reject data
|
||||
// from being saved in a snapshot based on path only
|
||||
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []RejectByNameFunc, err error) {
|
||||
func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (fs []archiver.RejectByNameFunc, err error) {
|
||||
// exclude restic cache
|
||||
if repo.Cache != nil {
|
||||
f, err := rejectResticCache(repo)
|
||||
|
|
@ -308,11 +309,13 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository) (
|
|||
fs = append(fs, f)
|
||||
}
|
||||
|
||||
fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
|
||||
fsPatterns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fs = append(fs, fsPatterns...)
|
||||
for _, pat := range fsPatterns {
|
||||
fs = append(fs, archiver.RejectByNameFunc(pat))
|
||||
}
|
||||
|
||||
return fs, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/restorer"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
|
|
@ -49,8 +50,8 @@ Exit status is 12 if the password is incorrect.
|
|||
|
||||
// RestoreOptions collects all options for the restore command.
|
||||
type RestoreOptions struct {
|
||||
excludePatternOptions
|
||||
includePatternOptions
|
||||
filter.ExcludePatternOptions
|
||||
filter.IncludePatternOptions
|
||||
Target string
|
||||
restic.SnapshotFilter
|
||||
DryRun bool
|
||||
|
|
@ -68,8 +69,8 @@ func init() {
|
|||
flags := cmdRestore.Flags()
|
||||
flags.StringVarP(&restoreOptions.Target, "target", "t", "", "directory to extract data to")
|
||||
|
||||
restoreOptions.excludePatternOptions.Add(flags)
|
||||
restoreOptions.includePatternOptions.Add(flags)
|
||||
restoreOptions.ExcludePatternOptions.Add(flags)
|
||||
restoreOptions.IncludePatternOptions.Add(flags)
|
||||
|
||||
initSingleSnapshotFilter(flags, &restoreOptions.SnapshotFilter)
|
||||
flags.BoolVar(&restoreOptions.DryRun, "dry-run", false, "do not write any data, just show what would be done")
|
||||
|
|
@ -82,12 +83,12 @@ func init() {
|
|||
func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
|
||||
term *termstatus.Terminal, args []string) error {
|
||||
|
||||
excludePatternFns, err := opts.excludePatternOptions.CollectPatterns()
|
||||
excludePatternFns, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
includePatternFns, err := opts.includePatternOptions.CollectPatterns()
|
||||
includePatternFns, err := opts.IncludePatternOptions.CollectPatterns(Warnf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/walker"
|
||||
|
|
@ -88,7 +89,7 @@ type RewriteOptions struct {
|
|||
|
||||
Metadata snapshotMetadataArgs
|
||||
restic.SnapshotFilter
|
||||
excludePatternOptions
|
||||
filter.ExcludePatternOptions
|
||||
}
|
||||
|
||||
var rewriteOptions RewriteOptions
|
||||
|
|
@ -103,7 +104,7 @@ func init() {
|
|||
f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
|
||||
|
||||
initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
|
||||
rewriteOptions.excludePatternOptions.Add(f)
|
||||
rewriteOptions.ExcludePatternOptions.Add(f)
|
||||
}
|
||||
|
||||
type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
|
||||
|
|
@ -113,7 +114,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
|
|||
return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
|
||||
}
|
||||
|
||||
rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
|
||||
rejectByNameFuncs, err := opts.ExcludePatternOptions.CollectPatterns(Warnf)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
|
@ -263,7 +264,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
|
|||
}
|
||||
|
||||
func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
|
||||
if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
|
||||
if opts.ExcludePatternOptions.Empty() && opts.Metadata.empty() {
|
||||
return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
|
|
@ -12,7 +13,7 @@ import (
|
|||
|
||||
func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
|
||||
opts := RewriteOptions{
|
||||
excludePatternOptions: excludePatternOptions{
|
||||
ExcludePatternOptions: filter.ExcludePatternOptions{
|
||||
Excludes: excludes,
|
||||
},
|
||||
Forget: forget,
|
||||
|
|
|
|||
|
|
@ -1,60 +1,16 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/archiver"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/restic/restic/internal/fs"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/textfile"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// RejectByNameFunc is a function that takes a filename of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectByNameFunc func(path string) bool
|
||||
|
||||
// rejectByPattern returns a RejectByNameFunc which rejects files that match
|
||||
// one of the patterns.
|
||||
func rejectByPattern(patterns []string) RejectByNameFunc {
|
||||
parsedPatterns := filter.ParsePatterns(patterns)
|
||||
return func(item string) bool {
|
||||
matched, err := filter.List(parsedPatterns, item)
|
||||
if err != nil {
|
||||
Warnf("error for exclude pattern: %v", err)
|
||||
}
|
||||
|
||||
if matched {
|
||||
debug.Log("path %q excluded by an exclude pattern", item)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// Same as `rejectByPattern` but case insensitive.
|
||||
func rejectByInsensitivePattern(patterns []string) RejectByNameFunc {
|
||||
for index, path := range patterns {
|
||||
patterns[index] = strings.ToLower(path)
|
||||
}
|
||||
|
||||
rejFunc := rejectByPattern(patterns)
|
||||
return func(item string) bool {
|
||||
return rejFunc(strings.ToLower(item))
|
||||
}
|
||||
}
|
||||
|
||||
// rejectResticCache returns a RejectByNameFunc that rejects the restic cache
|
||||
// directory (if set).
|
||||
func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
|
||||
func rejectResticCache(repo *repository.Repository) (archiver.RejectByNameFunc, error) {
|
||||
if repo.Cache == nil {
|
||||
return func(string) bool {
|
||||
return false
|
||||
|
|
@ -75,115 +31,3 @@ func rejectResticCache(repo *repository.Repository) (RejectByNameFunc, error) {
|
|||
return false
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readPatternsFromFiles reads all files and returns the list of
|
||||
// patterns. For each line, leading and trailing white space is removed
|
||||
// and comment lines are ignored. For each remaining pattern, environment
|
||||
// variables are resolved. For adding a literal dollar sign ($), write $$ to
|
||||
// the file.
|
||||
func readPatternsFromFiles(files []string) ([]string, error) {
|
||||
getenvOrDollar := func(s string) string {
|
||||
if s == "$" {
|
||||
return "$"
|
||||
}
|
||||
return os.Getenv(s)
|
||||
}
|
||||
|
||||
var patterns []string
|
||||
for _, filename := range files {
|
||||
err := func() (err error) {
|
||||
data, err := textfile.Read(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
// ignore empty lines
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// strip comments
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
line = os.Expand(line, getenvOrDollar)
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
return scanner.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
|
||||
}
|
||||
}
|
||||
return patterns, nil
|
||||
}
|
||||
|
||||
type excludePatternOptions struct {
|
||||
Excludes []string
|
||||
InsensitiveExcludes []string
|
||||
ExcludeFiles []string
|
||||
InsensitiveExcludeFiles []string
|
||||
}
|
||||
|
||||
func (opts *excludePatternOptions) Add(f *pflag.FlagSet) {
|
||||
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
|
||||
}
|
||||
|
||||
func (opts *excludePatternOptions) Empty() bool {
|
||||
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
|
||||
}
|
||||
|
||||
func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
|
||||
var fs []RejectByNameFunc
|
||||
// add patterns from file
|
||||
if len(opts.ExcludeFiles) > 0 {
|
||||
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := filter.ValidatePatterns(excludePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--exclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.Excludes = append(opts.Excludes, excludePatterns...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludeFiles) > 0 {
|
||||
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := filter.ValidatePatterns(excludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludes) > 0 {
|
||||
if err := filter.ValidatePatterns(opts.InsensitiveExcludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
|
||||
}
|
||||
|
||||
if len(opts.Excludes) > 0 {
|
||||
if err := filter.ValidatePatterns(opts.Excludes); err != nil {
|
||||
return nil, errors.Fatalf("--exclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, rejectByPattern(opts.Excludes))
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import (
|
|||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/filter"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
|
|
@ -17,14 +18,14 @@ func TestBackupFailsWhenUsingInvalidPatterns(t *testing.T) {
|
|||
var err error
|
||||
|
||||
// Test --exclude
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
// Test --iexclude
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
|
|
@ -47,14 +48,14 @@ func TestBackupFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
|
|||
var err error
|
||||
|
||||
// Test --exclude-file:
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{excludeFile}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
// Test --iexclude-file
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
|
||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{excludeFile}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
|
|
@ -70,28 +71,28 @@ func TestRestoreFailsWhenUsingInvalidPatterns(t *testing.T) {
|
|||
var err error
|
||||
|
||||
// Test --exclude
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{Excludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --exclude: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
// Test --iexclude
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --iexclude: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
// Test --include
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{Includes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --include: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
// Test --iinclude
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludes: []string{"*[._]log[.-][0-9]", "!*[._]log[.-][0-9]"}}}, env.gopts)
|
||||
|
||||
rtest.Equals(t, `Fatal: --iinclude: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
|
|
@ -111,22 +112,22 @@ func TestRestoreFailsWhenUsingInvalidPatternsFromFile(t *testing.T) {
|
|||
t.Fatalf("Could not write include file: %v", fileErr)
|
||||
}
|
||||
|
||||
err := testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
err := testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{IncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
rtest.Equals(t, `Fatal: --include-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{ExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
rtest.Equals(t, `Fatal: --exclude-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{includePatternOptions: includePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{IncludePatternOptions: filter.IncludePatternOptions{InsensitiveIncludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
rtest.Equals(t, `Fatal: --iinclude-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{excludePatternOptions: excludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
err = testRunRestoreAssumeFailure("latest", RestoreOptions{ExcludePatternOptions: filter.ExcludePatternOptions{InsensitiveExcludeFiles: []string{patternsFile}}}, env.gopts)
|
||||
rtest.Equals(t, `Fatal: --iexclude-file: invalid pattern(s) provided:
|
||||
*[._]log[.-][0-9]
|
||||
!*[._]log[.-][0-9]`, err.Error())
|
||||
|
|
|
|||
|
|
@ -14,6 +14,16 @@ import (
|
|||
"github.com/restic/restic/internal/ui"
|
||||
)
|
||||
|
||||
// RejectByNameFunc is a function that takes a filename of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectByNameFunc func(path string) bool
|
||||
|
||||
// RejectFunc is a function that takes a filename and os.FileInfo of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
|
||||
|
||||
type rejectionCache struct {
|
||||
m map[string]bool
|
||||
mtx sync.Mutex
|
||||
|
|
@ -49,11 +59,6 @@ func (rc *rejectionCache) Store(dir string, rejected bool) {
|
|||
rc.m[dir] = rejected
|
||||
}
|
||||
|
||||
// RejectFunc is a function that takes a filename and os.FileInfo of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectFunc func(path string, fi os.FileInfo, fs fs.FS) bool
|
||||
|
||||
// RejectIfPresent returns a RejectByNameFunc which itself returns whether a path
|
||||
// should be excluded. The RejectByNameFunc considers a file to be excluded when
|
||||
// it resides in a directory with an exclusion file, that is specified by
|
||||
|
|
|
|||
162
internal/filter/exclude.go
Normal file
162
internal/filter/exclude.go
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
package filter
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/textfile"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// RejectByNameFunc is a function that takes a filename of a
|
||||
// file that would be included in the backup. The function returns true if it
|
||||
// should be excluded (rejected) from the backup.
|
||||
type RejectByNameFunc func(path string) bool
|
||||
|
||||
// RejectByPattern returns a RejectByNameFunc which rejects files that match
|
||||
// one of the patterns.
|
||||
func RejectByPattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
|
||||
parsedPatterns := ParsePatterns(patterns)
|
||||
return func(item string) bool {
|
||||
matched, err := List(parsedPatterns, item)
|
||||
if err != nil {
|
||||
warnf("error for exclude pattern: %v", err)
|
||||
}
|
||||
|
||||
if matched {
|
||||
debug.Log("path %q excluded by an exclude pattern", item)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// RejectByInsensitivePattern is like RejectByPattern but case insensitive.
|
||||
func RejectByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) RejectByNameFunc {
|
||||
for index, path := range patterns {
|
||||
patterns[index] = strings.ToLower(path)
|
||||
}
|
||||
|
||||
rejFunc := RejectByPattern(patterns, warnf)
|
||||
return func(item string) bool {
|
||||
return rejFunc(strings.ToLower(item))
|
||||
}
|
||||
}
|
||||
|
||||
// readPatternsFromFiles reads all files and returns the list of
|
||||
// patterns. For each line, leading and trailing white space is removed
|
||||
// and comment lines are ignored. For each remaining pattern, environment
|
||||
// variables are resolved. For adding a literal dollar sign ($), write $$ to
|
||||
// the file.
|
||||
func readPatternsFromFiles(files []string) ([]string, error) {
|
||||
getenvOrDollar := func(s string) string {
|
||||
if s == "$" {
|
||||
return "$"
|
||||
}
|
||||
return os.Getenv(s)
|
||||
}
|
||||
|
||||
var patterns []string
|
||||
for _, filename := range files {
|
||||
err := func() (err error) {
|
||||
data, err := textfile.Read(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(bytes.NewReader(data))
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
// ignore empty lines
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// strip comments
|
||||
if strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
line = os.Expand(line, getenvOrDollar)
|
||||
patterns = append(patterns, line)
|
||||
}
|
||||
return scanner.Err()
|
||||
}()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read patterns from file %q: %w", filename, err)
|
||||
}
|
||||
}
|
||||
return patterns, nil
|
||||
}
|
||||
|
||||
type ExcludePatternOptions struct {
|
||||
Excludes []string
|
||||
InsensitiveExcludes []string
|
||||
ExcludeFiles []string
|
||||
InsensitiveExcludeFiles []string
|
||||
}
|
||||
|
||||
func (opts *ExcludePatternOptions) Add(f *pflag.FlagSet) {
|
||||
f.StringArrayVarP(&opts.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&opts.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
|
||||
}
|
||||
|
||||
func (opts *ExcludePatternOptions) Empty() bool {
|
||||
return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
|
||||
}
|
||||
|
||||
func (opts ExcludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]RejectByNameFunc, error) {
|
||||
var fs []RejectByNameFunc
|
||||
// add patterns from file
|
||||
if len(opts.ExcludeFiles) > 0 {
|
||||
excludePatterns, err := readPatternsFromFiles(opts.ExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(excludePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--exclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.Excludes = append(opts.Excludes, excludePatterns...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludeFiles) > 0 {
|
||||
excludes, err := readPatternsFromFiles(opts.InsensitiveExcludeFiles)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := ValidatePatterns(excludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude-file: %s", err)
|
||||
}
|
||||
|
||||
opts.InsensitiveExcludes = append(opts.InsensitiveExcludes, excludes...)
|
||||
}
|
||||
|
||||
if len(opts.InsensitiveExcludes) > 0 {
|
||||
if err := ValidatePatterns(opts.InsensitiveExcludes); err != nil {
|
||||
return nil, errors.Fatalf("--iexclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, RejectByInsensitivePattern(opts.InsensitiveExcludes, warnf))
|
||||
}
|
||||
|
||||
if len(opts.Excludes) > 0 {
|
||||
if err := ValidatePatterns(opts.Excludes); err != nil {
|
||||
return nil, errors.Fatalf("--exclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, RejectByPattern(opts.Excludes, warnf))
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package main
|
||||
package filter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
|
@ -21,7 +21,7 @@ func TestRejectByPattern(t *testing.T) {
|
|||
|
||||
for _, tc := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
reject := rejectByPattern(patterns)
|
||||
reject := RejectByPattern(patterns, nil)
|
||||
res := reject(tc.filename)
|
||||
if res != tc.reject {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
|
|
@ -48,7 +48,7 @@ func TestRejectByInsensitivePattern(t *testing.T) {
|
|||
|
||||
for _, tc := range tests {
|
||||
t.Run("", func(t *testing.T) {
|
||||
reject := rejectByInsensitivePattern(patterns)
|
||||
reject := RejectByInsensitivePattern(patterns, nil)
|
||||
res := reject(tc.filename)
|
||||
if res != tc.reject {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
|
|
@ -1,10 +1,9 @@
|
|||
package main
|
||||
package filter
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/filter"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
|
|
@ -12,21 +11,21 @@ import (
|
|||
// in the restore process and returns whether it should be included.
|
||||
type IncludeByNameFunc func(item string) (matched bool, childMayMatch bool)
|
||||
|
||||
type includePatternOptions struct {
|
||||
type IncludePatternOptions struct {
|
||||
Includes []string
|
||||
InsensitiveIncludes []string
|
||||
IncludeFiles []string
|
||||
InsensitiveIncludeFiles []string
|
||||
}
|
||||
|
||||
func (opts *includePatternOptions) Add(f *pflag.FlagSet) {
|
||||
func (opts *IncludePatternOptions) Add(f *pflag.FlagSet) {
|
||||
f.StringArrayVarP(&opts.Includes, "include", "i", nil, "include a `pattern` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveIncludes, "iinclude", nil, "same as --include `pattern` but ignores the casing of filenames")
|
||||
f.StringArrayVar(&opts.IncludeFiles, "include-file", nil, "read include patterns from a `file` (can be specified multiple times)")
|
||||
f.StringArrayVar(&opts.InsensitiveIncludeFiles, "iinclude-file", nil, "same as --include-file but ignores casing of `file`names in patterns")
|
||||
}
|
||||
|
||||
func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error) {
|
||||
func (opts IncludePatternOptions) CollectPatterns(warnf func(msg string, args ...interface{})) ([]IncludeByNameFunc, error) {
|
||||
var fs []IncludeByNameFunc
|
||||
if len(opts.IncludeFiles) > 0 {
|
||||
includePatterns, err := readPatternsFromFiles(opts.IncludeFiles)
|
||||
|
|
@ -34,7 +33,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := filter.ValidatePatterns(includePatterns); err != nil {
|
||||
if err := ValidatePatterns(includePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--include-file: %s", err)
|
||||
}
|
||||
|
||||
|
|
@ -47,7 +46,7 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
|
|||
return nil, err
|
||||
}
|
||||
|
||||
if err := filter.ValidatePatterns(includePatterns); err != nil {
|
||||
if err := ValidatePatterns(includePatterns); err != nil {
|
||||
return nil, errors.Fatalf("--iinclude-file: %s", err)
|
||||
}
|
||||
|
||||
|
|
@ -55,45 +54,45 @@ func (opts includePatternOptions) CollectPatterns() ([]IncludeByNameFunc, error)
|
|||
}
|
||||
|
||||
if len(opts.InsensitiveIncludes) > 0 {
|
||||
if err := filter.ValidatePatterns(opts.InsensitiveIncludes); err != nil {
|
||||
if err := ValidatePatterns(opts.InsensitiveIncludes); err != nil {
|
||||
return nil, errors.Fatalf("--iinclude: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, includeByInsensitivePattern(opts.InsensitiveIncludes))
|
||||
fs = append(fs, IncludeByInsensitivePattern(opts.InsensitiveIncludes, warnf))
|
||||
}
|
||||
|
||||
if len(opts.Includes) > 0 {
|
||||
if err := filter.ValidatePatterns(opts.Includes); err != nil {
|
||||
if err := ValidatePatterns(opts.Includes); err != nil {
|
||||
return nil, errors.Fatalf("--include: %s", err)
|
||||
}
|
||||
|
||||
fs = append(fs, includeByPattern(opts.Includes))
|
||||
fs = append(fs, IncludeByPattern(opts.Includes, warnf))
|
||||
}
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// includeByPattern returns a IncludeByNameFunc which includes files that match
|
||||
// IncludeByPattern returns a IncludeByNameFunc which includes files that match
|
||||
// one of the patterns.
|
||||
func includeByPattern(patterns []string) IncludeByNameFunc {
|
||||
parsedPatterns := filter.ParsePatterns(patterns)
|
||||
func IncludeByPattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
|
||||
parsedPatterns := ParsePatterns(patterns)
|
||||
return func(item string) (matched bool, childMayMatch bool) {
|
||||
matched, childMayMatch, err := filter.ListWithChild(parsedPatterns, item)
|
||||
matched, childMayMatch, err := ListWithChild(parsedPatterns, item)
|
||||
if err != nil {
|
||||
Warnf("error for include pattern: %v", err)
|
||||
warnf("error for include pattern: %v", err)
|
||||
}
|
||||
|
||||
return matched, childMayMatch
|
||||
}
|
||||
}
|
||||
|
||||
// includeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
|
||||
// IncludeByInsensitivePattern returns a IncludeByNameFunc which includes files that match
|
||||
// one of the patterns, ignoring the casing of the filenames.
|
||||
func includeByInsensitivePattern(patterns []string) IncludeByNameFunc {
|
||||
func IncludeByInsensitivePattern(patterns []string, warnf func(msg string, args ...interface{})) IncludeByNameFunc {
|
||||
for index, path := range patterns {
|
||||
patterns[index] = strings.ToLower(path)
|
||||
}
|
||||
|
||||
includeFunc := includeByPattern(patterns)
|
||||
includeFunc := IncludeByPattern(patterns, warnf)
|
||||
return func(item string) (matched bool, childMayMatch bool) {
|
||||
return includeFunc(strings.ToLower(item))
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
package main
|
||||
package filter
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
|
@ -21,7 +21,7 @@ func TestIncludeByPattern(t *testing.T) {
|
|||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.filename, func(t *testing.T) {
|
||||
includeFunc := includeByPattern(patterns)
|
||||
includeFunc := IncludeByPattern(patterns, nil)
|
||||
matched, _ := includeFunc(tc.filename)
|
||||
if matched != tc.include {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
|
|
@ -48,7 +48,7 @@ func TestIncludeByInsensitivePattern(t *testing.T) {
|
|||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.filename, func(t *testing.T) {
|
||||
includeFunc := includeByInsensitivePattern(patterns)
|
||||
includeFunc := IncludeByInsensitivePattern(patterns, nil)
|
||||
matched, _ := includeFunc(tc.filename)
|
||||
if matched != tc.include {
|
||||
t.Fatalf("wrong result for filename %v: want %v, got %v",
|
||||
Loading…
Reference in a new issue