From dc29709742b97e252ad2fa107ae01028e749e33e Mon Sep 17 00:00:00 2001
From: Dmitry Nezhevenko <dion@dion.org.ua>
Date: Tue, 5 May 2020 22:03:57 +0300
Subject: [PATCH 01/25] Implement 'rewrite' command to exclude files from
 existing snapshots

---
 changelog/unreleased/issue-14 |   3 +
 cmd/restic/cmd_rewrite.go     | 296 ++++++++++++++++++++++++++++++++++
 2 files changed, 299 insertions(+)
 create mode 100644 changelog/unreleased/issue-14
 create mode 100644 cmd/restic/cmd_rewrite.go

diff --git a/changelog/unreleased/issue-14 b/changelog/unreleased/issue-14
new file mode 100644
index 000000000..a52b8a1a3
--- /dev/null
+++ b/changelog/unreleased/issue-14
@@ -0,0 +1,3 @@
+Change: Implement rewrite command
+
+TODO: write here
diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
new file mode 100644
index 000000000..4734c96e8
--- /dev/null
+++ b/cmd/restic/cmd_rewrite.go
@@ -0,0 +1,296 @@
+package main
+
+import (
+	"context"
+	"path"
+
+	"github.com/spf13/cobra"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+)
+
+var cmdRewrite = &cobra.Command{
+	Use:   "rewrite [f] [all|snapshotID ...]",
+	Short: "Modify existing snapshots by deleting files",
+	Long: `
+The "rewrite" command excludes files from existing snapshots.
+
+By default 'rewrite' will create new snapshot that will contains same data as
+source snapshot except excluded data. All metadata (time, host, tags) will be preserved.
+Special tag 'rewrite' will be added to new snapshot to distinguish it from source
+(unless --inplace is used)
+
+If --inplace option is used, old snapshot will be removed from repository.
+
+Snapshots to rewrite are specified using --host, --tag, --path or by providing list of snapshotID.
+Alternatively it's possible to use special 'all' snapshot that will match all snapshots
+
+Please note, that this command only modifies snapshot objects. In order to delete
+data from repository use 'prune' command
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runRewrite(rewriteOptions, globalOptions, args)
+	},
+}
+
+// RewriteOptions collects all options for the ls command.
+type RewriteOptions struct {
+	// TagOptions bundles all options for the 'tag' command.
+	Hosts   []string
+	Paths   []string
+	Tags    restic.TagLists
+	Inplace bool
+	DryRun  bool
+
+	// Exclude options
+	Excludes            []string
+	InsensitiveExcludes []string
+	ExcludeFiles        []string
+}
+
+var rewriteOptions RewriteOptions
+
+func init() {
+	cmdRoot.AddCommand(cmdRewrite)
+
+	f := cmdRewrite.Flags()
+	f.StringArrayVarP(&rewriteOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
+	f.Var(&rewriteOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
+	f.StringArrayVar(&rewriteOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
+	f.BoolVarP(&rewriteOptions.Inplace, "inplace", "", false, "replace existing snapshots")
+	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+
+	// Excludes
+	f.StringArrayVarP(&rewriteOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
+	f.StringArrayVar(&rewriteOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
+	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
+}
+
+type saveTreeFunction = func(*restic.Tree) (restic.ID, error)
+
+func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc, saveTreeFunc saveTreeFunction) (newNodeID restic.ID, err error) {
+	curTree, err := repo.LoadTree(ctx, nodeID)
+	if err != nil {
+		return nodeID, err
+	}
+
+	debug.Log("filterNode: %s, nodeId: %s\n", nodepath, nodeID.Str())
+
+	changed := false
+	newTree := restic.NewTree()
+	for _, node := range curTree.Nodes {
+		path := path.Join(nodepath, node.Name)
+		if !checkExclude(path) {
+			if node.Subtree == nil {
+				newTree.Insert(node)
+				continue
+			}
+			newNode := node
+			newID, err := filterNode(ctx, repo, path, *node.Subtree, checkExclude, saveTreeFunc)
+			if err != nil {
+				return nodeID, err
+			}
+			if newID == *node.Subtree {
+				newTree.Insert(node)
+			} else {
+				changed = true
+				newNode.Subtree = new(restic.ID)
+				*newNode.Subtree = newID
+				newTree.Insert(newNode)
+			}
+		} else {
+			Verbosef("excluding %s\n", path)
+			changed = true
+		}
+	}
+
+	if changed {
+		// Save new tree
+		newTreeID, err := saveTreeFunc(newTree)
+		debug.Log("filterNode: save new tree for %s as %v\n", nodepath, newTreeID)
+		return newTreeID, err
+	}
+
+	return nodeID, nil
+}
+
+func collectRejectFuncsForRewrite(opts RewriteOptions) (fs []RejectByNameFunc, err error) {
+	//TODO: merge with cmd_backup
+
+	// add patterns from file
+	if len(opts.ExcludeFiles) > 0 {
+		excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
+		if err != nil {
+			return nil, err
+		}
+		opts.Excludes = append(opts.Excludes, excludes...)
+	}
+
+	if len(opts.InsensitiveExcludes) > 0 {
+		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
+	}
+
+	if len(opts.Excludes) > 0 {
+		fs = append(fs, rejectByPattern(opts.Excludes))
+	}
+
+	return fs, nil
+}
+
+func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions, gopts GlobalOptions) (bool, error) {
+	if sn.Tree == nil {
+		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
+	}
+
+	var saveTreeFunc saveTreeFunction
+	if !opts.DryRun {
+		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
+			return repo.SaveTree(ctx, tree)
+		}
+	} else {
+		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
+			return restic.ID{0}, nil
+		}
+	}
+
+	rejectByNameFuncs, err := collectRejectFuncsForRewrite(opts)
+	if err != nil {
+		return false, err
+	}
+
+	checkExclude := func(nodepath string) bool {
+		for _, reject := range rejectByNameFuncs {
+			if reject(nodepath) {
+				return true
+			}
+		}
+		return false
+	}
+
+	filteredTree, err := filterNode(ctx, repo, "/", *sn.Tree, checkExclude, saveTreeFunc)
+
+	if err != nil {
+		return false, err
+	}
+
+	if filteredTree == *sn.Tree {
+		debug.Log("Snapshot not touched\n")
+		return false, nil
+	}
+
+	debug.Log("Snapshot modified\n")
+	if opts.DryRun {
+		Printf("Will modify snapshot: %s\n", sn.String())
+		return true, nil
+	}
+
+	err = repo.Flush(ctx)
+	if err != nil {
+		return false, err
+	}
+	err = repo.SaveIndex(ctx)
+	if err != nil {
+		return false, err
+	}
+
+	// Retain the original snapshot id over all tag changes.
+	if sn.Original == nil {
+		sn.Original = sn.ID()
+	}
+	*sn.Tree = filteredTree
+
+	if !opts.Inplace {
+		sn.AddTags([]string{"rewrite"})
+	}
+
+	// Save the new snapshot.
+	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+	if err != nil {
+		return false, err
+	}
+	err = repo.Flush(ctx)
+	if err != nil {
+		return true, err
+	}
+
+	if opts.Inplace {
+		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
+		if err = repo.Backend().Remove(ctx, h); err != nil {
+			return false, err
+		}
+
+		debug.Log("old snapshot %v removed", sn.ID())
+	}
+	Printf("new snapshot saved as %v\n", id)
+	return true, nil
+}
+
+func runRewrite(opts RewriteOptions, gopts GlobalOptions, args []string) error {
+
+	if len(opts.Hosts) == 0 && len(opts.Tags) == 0 && len(opts.Paths) == 0 && len(args) == 0 {
+		return errors.Fatal("no snapshots provided")
+	}
+
+	if len(opts.ExcludeFiles) == 0 && len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 {
+		return errors.Fatal("Nothing to do: no excludes provided")
+	}
+
+	if len(args) == 1 && args[0] == "all" {
+		args = []string{}
+	}
+
+	repo, err := OpenRepository(gopts)
+	if err != nil {
+		return err
+	}
+
+	if !gopts.NoLock && !opts.DryRun {
+		Verbosef("create exclusive lock for repository\n")
+		lock, err := lockRepoExclusive(repo)
+		defer unlockRepo(lock)
+		if err != nil {
+			return err
+		}
+	}
+
+	if err = repo.LoadIndex(gopts.ctx); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(gopts.ctx)
+	defer cancel()
+
+	changedCount := 0
+	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+		Verbosef("Checking snapshot %s\n", sn.String())
+		changed, err := rewriteSnapshot(ctx, repo, sn, opts, gopts)
+		if err != nil {
+			Warnf("unable to rewrite snapshot ID %q, ignoring: %v\n", sn.ID(), err)
+			continue
+		}
+		if changed {
+			changedCount++
+		}
+	}
+
+	if changedCount == 0 {
+		Verbosef("no snapshots modified\n")
+	} else {
+		if !opts.DryRun {
+			Verbosef("modified %v snapshots\n", changedCount)
+		} else {
+			Verbosef("dry run. %v snapshots affected\n", changedCount)
+		}
+	}
+
+	return nil
+}

From b9227743431e2f2b6f34dc754acada7e3c3cbb17 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 21:48:20 +0200
Subject: [PATCH 02/25] rewrite: fix compilation

---
 cmd/restic/cmd_rewrite.go | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 4734c96e8..58f2949f6 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -38,7 +39,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRewrite(rewriteOptions, globalOptions, args)
+		return runRewrite(cmd.Context(), rewriteOptions, globalOptions, args)
 	},
 }
 
@@ -78,7 +79,7 @@ func init() {
 type saveTreeFunction = func(*restic.Tree) (restic.ID, error)
 
 func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc, saveTreeFunc saveTreeFunction) (newNodeID restic.ID, err error) {
-	curTree, err := repo.LoadTree(ctx, nodeID)
+	curTree, err := restic.LoadTree(ctx, repo, nodeID)
 	if err != nil {
 		return nodeID, err
 	}
@@ -86,7 +87,7 @@ func filterNode(ctx context.Context, repo restic.Repository, nodepath string, no
 	debug.Log("filterNode: %s, nodeId: %s\n", nodepath, nodeID.Str())
 
 	changed := false
-	newTree := restic.NewTree()
+	newTree := restic.NewTree(len(curTree.Nodes))
 	for _, node := range curTree.Nodes {
 		path := path.Join(nodepath, node.Name)
 		if !checkExclude(path) {
@@ -154,7 +155,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	var saveTreeFunc saveTreeFunction
 	if !opts.DryRun {
 		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
-			return repo.SaveTree(ctx, tree)
+			return restic.SaveTree(ctx, repo, tree)
 		}
 	} else {
 		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
@@ -197,10 +198,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	if err != nil {
 		return false, err
 	}
-	err = repo.SaveIndex(ctx)
-	if err != nil {
-		return false, err
-	}
 
 	// Retain the original snapshot id over all tag changes.
 	if sn.Original == nil {
@@ -213,7 +210,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	}
 
 	// Save the new snapshot.
-	id, err := repo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
+	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
 		return false, err
 	}
@@ -234,7 +231,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	return true, nil
 }
 
-func runRewrite(opts RewriteOptions, gopts GlobalOptions, args []string) error {
+func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
 
 	if len(opts.Hosts) == 0 && len(opts.Tags) == 0 && len(opts.Paths) == 0 && len(args) == 0 {
 		return errors.Fatal("no snapshots provided")
@@ -248,29 +245,32 @@ func runRewrite(opts RewriteOptions, gopts GlobalOptions, args []string) error {
 		args = []string{}
 	}
 
-	repo, err := OpenRepository(gopts)
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err
 	}
 
 	if !gopts.NoLock && !opts.DryRun {
 		Verbosef("create exclusive lock for repository\n")
-		lock, err := lockRepoExclusive(repo)
+		var lock *restic.Lock
+		lock, ctx, err = lockRepoExclusive(ctx, repo)
 		defer unlockRepo(lock)
 		if err != nil {
 			return err
 		}
 	}
 
-	if err = repo.LoadIndex(gopts.ctx); err != nil {
+	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	if err != nil {
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(gopts.ctx)
-	defer cancel()
+	if err = repo.LoadIndex(ctx); err != nil {
+		return err
+	}
 
 	changedCount := 0
-	for sn := range FindFilteredSnapshots(ctx, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
+	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
 		Verbosef("Checking snapshot %s\n", sn.String())
 		changed, err := rewriteSnapshot(ctx, repo, sn, opts, gopts)
 		if err != nil {

From 82592b88b5da12fdbd8ecaf7c19ec82adb277519 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:00:37 +0200
Subject: [PATCH 03/25] rewrite: address most review comments

---
 changelog/unreleased/issue-14 |  8 ++++++--
 cmd/restic/cmd_rewrite.go     | 30 +++++++++++++-----------------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/changelog/unreleased/issue-14 b/changelog/unreleased/issue-14
index a52b8a1a3..93f83686d 100644
--- a/changelog/unreleased/issue-14
+++ b/changelog/unreleased/issue-14
@@ -1,3 +1,7 @@
-Change: Implement rewrite command
+Enhancement: Implement rewrite command
 
-TODO: write here
+We've added a new command which allows to rewrite existing snapshots to remove
+unwanted files.
+
+https://github.com/restic/restic/issues/14
+https://github.com/restic/restic/pull/2731
diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 58f2949f6..a5460fd93 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -14,23 +14,24 @@ import (
 )
 
 var cmdRewrite = &cobra.Command{
-	Use:   "rewrite [f] [all|snapshotID ...]",
-	Short: "Modify existing snapshots by deleting files",
+	Use:   "rewrite [flags] [all|snapshotID ...]",
+	Short: "Rewrite existing snapshots",
 	Long: `
 The "rewrite" command excludes files from existing snapshots.
 
-By default 'rewrite' will create new snapshot that will contains same data as
-source snapshot except excluded data. All metadata (time, host, tags) will be preserved.
-Special tag 'rewrite' will be added to new snapshot to distinguish it from source
-(unless --inplace is used)
+By default 'rewrite' will create new snapshots that will contains same data as
+the source snapshots but without excluded files. All metadata (time, host, tags)
+will be preserved. The special tag 'rewrite' will be added to new snapshots to
+distinguish it from the source (unless --inplace is used).
 
 If --inplace option is used, old snapshot will be removed from repository.
 
-Snapshots to rewrite are specified using --host, --tag, --path or by providing list of snapshotID.
-Alternatively it's possible to use special 'all' snapshot that will match all snapshots
+Snapshots to rewrite are specified using --host, --tag, --path or by providing
+a list of snapshot ids. Alternatively it's possible to use special snapshot id 'all'
+that will match all snapshots.
 
-Please note, that this command only modifies snapshot objects. In order to delete
-data from repository use 'prune' command
+Please note, that this command only creates new snapshots. In order to delete
+data from the repository use 'prune' command.
 
 EXIT STATUS
 ===========
@@ -43,9 +44,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-// RewriteOptions collects all options for the ls command.
+// RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
-	// TagOptions bundles all options for the 'tag' command.
 	Hosts   []string
 	Paths   []string
 	Tags    restic.TagLists
@@ -214,10 +214,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	if err != nil {
 		return false, err
 	}
-	err = repo.Flush(ctx)
-	if err != nil {
-		return true, err
-	}
 
 	if opts.Inplace {
 		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
@@ -250,7 +246,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 		return err
 	}
 
-	if !gopts.NoLock && !opts.DryRun {
+	if !opts.DryRun {
 		Verbosef("create exclusive lock for repository\n")
 		var lock *restic.Lock
 		lock, ctx, err = lockRepoExclusive(ctx, repo)

From 4d6ab83019db38e592d6695476f82d7a924eab4c Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:19:20 +0200
Subject: [PATCH 04/25] rewrite: use treejsonbuilder

---
 cmd/restic/cmd_rewrite.go | 37 +++++++++++++++++++++----------------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index a5460fd93..3172b6053 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -76,7 +76,7 @@ func init() {
 	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 }
 
-type saveTreeFunction = func(*restic.Tree) (restic.ID, error)
+type saveTreeFunction func([]byte) (restic.ID, error)
 
 func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc, saveTreeFunc saveTreeFunction) (newNodeID restic.ID, err error) {
 	curTree, err := restic.LoadTree(ctx, repo, nodeID)
@@ -87,26 +87,25 @@ func filterNode(ctx context.Context, repo restic.Repository, nodepath string, no
 	debug.Log("filterNode: %s, nodeId: %s\n", nodepath, nodeID.Str())
 
 	changed := false
-	newTree := restic.NewTree(len(curTree.Nodes))
+	tb := restic.NewTreeJSONBuilder()
 	for _, node := range curTree.Nodes {
 		path := path.Join(nodepath, node.Name)
 		if !checkExclude(path) {
 			if node.Subtree == nil {
-				newTree.Insert(node)
+				tb.AddNode(node)
 				continue
 			}
-			newNode := node
 			newID, err := filterNode(ctx, repo, path, *node.Subtree, checkExclude, saveTreeFunc)
 			if err != nil {
-				return nodeID, err
+				return restic.ID{}, err
 			}
-			if newID == *node.Subtree {
-				newTree.Insert(node)
-			} else {
+			if !node.Subtree.Equal(newID) {
 				changed = true
-				newNode.Subtree = new(restic.ID)
-				*newNode.Subtree = newID
-				newTree.Insert(newNode)
+			}
+			node.Subtree = &newID
+			err = tb.AddNode(node)
+			if err != nil {
+				return restic.ID{}, err
 			}
 		} else {
 			Verbosef("excluding %s\n", path)
@@ -115,8 +114,13 @@ func filterNode(ctx context.Context, repo restic.Repository, nodepath string, no
 	}
 
 	if changed {
+		tree, err := tb.Finalize()
+		if err != nil {
+			return restic.ID{}, err
+		}
+
 		// Save new tree
-		newTreeID, err := saveTreeFunc(newTree)
+		newTreeID, err := saveTreeFunc(tree)
 		debug.Log("filterNode: save new tree for %s as %v\n", nodepath, newTreeID)
 		return newTreeID, err
 	}
@@ -154,12 +158,13 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 	var saveTreeFunc saveTreeFunction
 	if !opts.DryRun {
-		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
-			return restic.SaveTree(ctx, repo, tree)
+		saveTreeFunc = func(tree []byte) (restic.ID, error) {
+			id, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
+			return id, err
 		}
 	} else {
-		saveTreeFunc = func(tree *restic.Tree) (restic.ID, error) {
-			return restic.ID{0}, nil
+		saveTreeFunc = func(tree []byte) (restic.ID, error) {
+			return restic.ID{}, nil
 		}
 	}
 

From c0f7ba2388aca69d8bf2354e3b6b77a229716084 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:22:09 +0200
Subject: [PATCH 05/25] rewrite: simplify dryrun

---
 cmd/restic/cmd_rewrite.go | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 3172b6053..015e66ddc 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -76,9 +76,7 @@ func init() {
 	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 }
 
-type saveTreeFunction func([]byte) (restic.ID, error)
-
-func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc, saveTreeFunc saveTreeFunction) (newNodeID restic.ID, err error) {
+func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc) (newNodeID restic.ID, err error) {
 	curTree, err := restic.LoadTree(ctx, repo, nodeID)
 	if err != nil {
 		return nodeID, err
@@ -95,7 +93,7 @@ func filterNode(ctx context.Context, repo restic.Repository, nodepath string, no
 				tb.AddNode(node)
 				continue
 			}
-			newID, err := filterNode(ctx, repo, path, *node.Subtree, checkExclude, saveTreeFunc)
+			newID, err := filterNode(ctx, repo, path, *node.Subtree, checkExclude)
 			if err != nil {
 				return restic.ID{}, err
 			}
@@ -120,7 +118,7 @@ func filterNode(ctx context.Context, repo restic.Repository, nodepath string, no
 		}
 
 		// Save new tree
-		newTreeID, err := saveTreeFunc(tree)
+		newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
 		debug.Log("filterNode: save new tree for %s as %v\n", nodepath, newTreeID)
 		return newTreeID, err
 	}
@@ -156,18 +154,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
 
-	var saveTreeFunc saveTreeFunction
-	if !opts.DryRun {
-		saveTreeFunc = func(tree []byte) (restic.ID, error) {
-			id, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
-			return id, err
-		}
-	} else {
-		saveTreeFunc = func(tree []byte) (restic.ID, error) {
-			return restic.ID{}, nil
-		}
-	}
-
 	rejectByNameFuncs, err := collectRejectFuncsForRewrite(opts)
 	if err != nil {
 		return false, err
@@ -182,7 +168,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false
 	}
 
-	filteredTree, err := filterNode(ctx, repo, "/", *sn.Tree, checkExclude, saveTreeFunc)
+	filteredTree, err := filterNode(ctx, repo, "/", *sn.Tree, checkExclude)
 
 	if err != nil {
 		return false, err
@@ -259,6 +245,8 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 		if err != nil {
 			return err
 		}
+	} else {
+		repo.SetDryRun()
 	}
 
 	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)

From f6339b88af52ee226d1dbd5fcffc4a2cbf769b5c Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:30:45 +0200
Subject: [PATCH 06/25] rewrite: extract tree filtering

---
 cmd/restic/cmd_rewrite.go   | 66 ++++++----------------------------
 internal/walker/rewriter.go | 72 +++++++++++++++++++++++++++++++++++++
 2 files changed, 82 insertions(+), 56 deletions(-)
 create mode 100644 internal/walker/rewriter.go

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 015e66ddc..677cc3bfd 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -2,7 +2,7 @@ package main
 
 import (
 	"context"
-	"path"
+	"fmt"
 
 	"github.com/spf13/cobra"
 
@@ -11,6 +11,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/walker"
 )
 
 var cmdRewrite = &cobra.Command{
@@ -76,56 +77,6 @@ func init() {
 	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
 }
 
-func filterNode(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, checkExclude RejectByNameFunc) (newNodeID restic.ID, err error) {
-	curTree, err := restic.LoadTree(ctx, repo, nodeID)
-	if err != nil {
-		return nodeID, err
-	}
-
-	debug.Log("filterNode: %s, nodeId: %s\n", nodepath, nodeID.Str())
-
-	changed := false
-	tb := restic.NewTreeJSONBuilder()
-	for _, node := range curTree.Nodes {
-		path := path.Join(nodepath, node.Name)
-		if !checkExclude(path) {
-			if node.Subtree == nil {
-				tb.AddNode(node)
-				continue
-			}
-			newID, err := filterNode(ctx, repo, path, *node.Subtree, checkExclude)
-			if err != nil {
-				return restic.ID{}, err
-			}
-			if !node.Subtree.Equal(newID) {
-				changed = true
-			}
-			node.Subtree = &newID
-			err = tb.AddNode(node)
-			if err != nil {
-				return restic.ID{}, err
-			}
-		} else {
-			Verbosef("excluding %s\n", path)
-			changed = true
-		}
-	}
-
-	if changed {
-		tree, err := tb.Finalize()
-		if err != nil {
-			return restic.ID{}, err
-		}
-
-		// Save new tree
-		newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
-		debug.Log("filterNode: save new tree for %s as %v\n", nodepath, newTreeID)
-		return newTreeID, err
-	}
-
-	return nodeID, nil
-}
-
 func collectRejectFuncsForRewrite(opts RewriteOptions) (fs []RejectByNameFunc, err error) {
 	//TODO: merge with cmd_backup
 
@@ -168,20 +119,23 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false
 	}
 
-	filteredTree, err := filterNode(ctx, repo, "/", *sn.Tree, checkExclude)
+	filteredTree, err := walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
+		CheckExclude: checkExclude,
+		PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
+	})
 
 	if err != nil {
 		return false, err
 	}
 
 	if filteredTree == *sn.Tree {
-		debug.Log("Snapshot not touched\n")
+		debug.Log("Snapshot %v not modified", sn)
 		return false, nil
 	}
 
-	debug.Log("Snapshot modified\n")
+	debug.Log("Snapshot %v modified", sn)
 	if opts.DryRun {
-		Printf("Will modify snapshot: %s\n", sn.String())
+		Printf("Would modify snapshot: %s\n", sn.String())
 		return true, nil
 	}
 
@@ -277,7 +231,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 		if !opts.DryRun {
 			Verbosef("modified %v snapshots\n", changedCount)
 		} else {
-			Verbosef("dry run. %v snapshots affected\n", changedCount)
+			Verbosef("dry run. would modify %v snapshots\n", changedCount)
 		}
 	}
 
diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go
new file mode 100644
index 000000000..e4e049da5
--- /dev/null
+++ b/internal/walker/rewriter.go
@@ -0,0 +1,72 @@
+package walker
+
+import (
+	"context"
+	"path"
+
+	"github.com/restic/restic/internal/debug"
+	"github.com/restic/restic/internal/restic"
+)
+
+type RejectByNameFunc func(path string) bool
+
+type TreeFilterVisitor struct {
+	CheckExclude RejectByNameFunc
+	PrintExclude func(string)
+}
+
+func FilterTree(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) {
+	curTree, err := restic.LoadTree(ctx, repo, nodeID)
+	if err != nil {
+		return restic.ID{}, err
+	}
+
+	debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
+
+	changed := false
+	tb := restic.NewTreeJSONBuilder()
+	for _, node := range curTree.Nodes {
+		path := path.Join(nodepath, node.Name)
+		if visitor.CheckExclude(path) {
+			if visitor.PrintExclude != nil {
+				visitor.PrintExclude(path)
+			}
+			changed = true
+			continue
+		}
+
+		if node.Subtree == nil {
+			err = tb.AddNode(node)
+			if err != nil {
+				return restic.ID{}, err
+			}
+			continue
+		}
+		newID, err := FilterTree(ctx, repo, path, *node.Subtree, visitor)
+		if err != nil {
+			return restic.ID{}, err
+		}
+		if !node.Subtree.Equal(newID) {
+			changed = true
+		}
+		node.Subtree = &newID
+		err = tb.AddNode(node)
+		if err != nil {
+			return restic.ID{}, err
+		}
+	}
+
+	if changed {
+		tree, err := tb.Finalize()
+		if err != nil {
+			return restic.ID{}, err
+		}
+
+		// Save new tree
+		newTreeID, _, _, err := repo.SaveBlob(ctx, restic.TreeBlob, tree, restic.ID{}, false)
+		debug.Log("filterTree: save new tree for %s as %v\n", nodepath, newTreeID)
+		return newTreeID, err
+	}
+
+	return nodeID, nil
+}

From 2b69a1c53b191d91eb688ede09702f9b63bd9691 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:37:56 +0200
Subject: [PATCH 07/25] rewrite: filter all snapshots if none are specified

---
 cmd/restic/cmd_rewrite.go | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 677cc3bfd..7213daedb 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -15,7 +15,7 @@ import (
 )
 
 var cmdRewrite = &cobra.Command{
-	Use:   "rewrite [flags] [all|snapshotID ...]",
+	Use:   "rewrite [flags] [snapshotID ...]",
 	Short: "Rewrite existing snapshots",
 	Long: `
 The "rewrite" command excludes files from existing snapshots.
@@ -28,8 +28,7 @@ distinguish it from the source (unless --inplace is used).
 If --inplace option is used, old snapshot will be removed from repository.
 
 Snapshots to rewrite are specified using --host, --tag, --path or by providing
-a list of snapshot ids. Alternatively it's possible to use special snapshot id 'all'
-that will match all snapshots.
+a list of snapshot ids. Not specifying a snapshot id will rewrite all snapshots.
 
 Please note, that this command only creates new snapshots. In order to delete
 data from the repository use 'prune' command.
@@ -174,18 +173,10 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
 
-	if len(opts.Hosts) == 0 && len(opts.Tags) == 0 && len(opts.Paths) == 0 && len(args) == 0 {
-		return errors.Fatal("no snapshots provided")
-	}
-
 	if len(opts.ExcludeFiles) == 0 && len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 {
 		return errors.Fatal("Nothing to do: no excludes provided")
 	}
 
-	if len(args) == 1 && args[0] == "all" {
-		args = []string{}
-	}
-
 	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
 		return err

From 4cace1ffe94616c4d2f4844fbf1f4d708722d26a Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 6 Sep 2022 22:56:29 +0200
Subject: [PATCH 08/25] unify exclude patterns with backup command

---
 cmd/restic/cmd_rewrite.go | 33 ++++++---------------------------
 1 file changed, 6 insertions(+), 27 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 7213daedb..aedda03d1 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -53,9 +53,10 @@ type RewriteOptions struct {
 	DryRun  bool
 
 	// Exclude options
-	Excludes            []string
-	InsensitiveExcludes []string
-	ExcludeFiles        []string
+	Excludes                []string
+	InsensitiveExcludes     []string
+	ExcludeFiles            []string
+	InsensitiveExcludeFiles []string
 }
 
 var rewriteOptions RewriteOptions
@@ -74,29 +75,7 @@ func init() {
 	f.StringArrayVarP(&rewriteOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
 	f.StringArrayVar(&rewriteOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
 	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-}
-
-func collectRejectFuncsForRewrite(opts RewriteOptions) (fs []RejectByNameFunc, err error) {
-	//TODO: merge with cmd_backup
-
-	// add patterns from file
-	if len(opts.ExcludeFiles) > 0 {
-		excludes, err := readExcludePatternsFromFiles(opts.ExcludeFiles)
-		if err != nil {
-			return nil, err
-		}
-		opts.Excludes = append(opts.Excludes, excludes...)
-	}
-
-	if len(opts.InsensitiveExcludes) > 0 {
-		fs = append(fs, rejectByInsensitivePattern(opts.InsensitiveExcludes))
-	}
-
-	if len(opts.Excludes) > 0 {
-		fs = append(fs, rejectByPattern(opts.Excludes))
-	}
-
-	return fs, nil
+	f.StringArrayVar(&rewriteOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
 }
 
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions, gopts GlobalOptions) (bool, error) {
@@ -104,7 +83,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
 
-	rejectByNameFuncs, err := collectRejectFuncsForRewrite(opts)
+	rejectByNameFuncs, err := collectExcludePatterns(opts.Excludes, opts.InsensitiveExcludes, opts.ExcludeFiles, opts.InsensitiveExcludeFiles)
 	if err != nil {
 		return false, err
 	}

From 559acea0d8557a4eac940a96e2ceaa35eae7fa7c Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Wed, 7 Sep 2022 23:01:45 +0200
Subject: [PATCH 09/25] unify exclude pattern options

---
 cmd/restic/cmd_rewrite.go | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index aedda03d1..bf256bbd7 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -52,11 +52,7 @@ type RewriteOptions struct {
 	Inplace bool
 	DryRun  bool
 
-	// Exclude options
-	Excludes                []string
-	InsensitiveExcludes     []string
-	ExcludeFiles            []string
-	InsensitiveExcludeFiles []string
+	excludePatternOptions
 }
 
 var rewriteOptions RewriteOptions
@@ -71,11 +67,7 @@ func init() {
 	f.BoolVarP(&rewriteOptions.Inplace, "inplace", "", false, "replace existing snapshots")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
 
-	// Excludes
-	f.StringArrayVarP(&rewriteOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)")
-	f.StringArrayVar(&rewriteOptions.InsensitiveExcludes, "iexclude", nil, "same as --exclude `pattern` but ignores the casing of filenames")
-	f.StringArrayVar(&rewriteOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)")
-	f.StringArrayVar(&rewriteOptions.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
+	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions, gopts GlobalOptions) (bool, error) {
@@ -83,7 +75,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
 
-	rejectByNameFuncs, err := collectExcludePatterns(opts.Excludes, opts.InsensitiveExcludes, opts.ExcludeFiles, opts.InsensitiveExcludeFiles)
+	rejectByNameFuncs, err := collectExcludePatterns(opts.excludePatternOptions)
 	if err != nil {
 		return false, err
 	}

From 7ebaf6e899373a1f3bca675e208137b957269444 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 9 Sep 2022 22:32:39 +0200
Subject: [PATCH 10/25] rewrite: start repository uploader goroutines

---
 cmd/restic/cmd_rewrite.go | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index bf256bbd7..9e3ab370d 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
@@ -89,11 +90,22 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false
 	}
 
-	filteredTree, err := walker.FilterTree(ctx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
-		CheckExclude: checkExclude,
-		PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
-	})
+	wg, wgCtx := errgroup.WithContext(ctx)
+	repo.StartPackUploader(wgCtx, wg)
 
+	var filteredTree restic.ID
+	wg.Go(func() error {
+		filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
+			CheckExclude: checkExclude,
+			PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
+		})
+		if err != nil {
+			return err
+		}
+
+		return repo.Flush(wgCtx)
+	})
+	err = wg.Wait()
 	if err != nil {
 		return false, err
 	}
@@ -109,11 +121,6 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return true, nil
 	}
 
-	err = repo.Flush(ctx)
-	if err != nil {
-		return false, err
-	}
-
 	// Retain the original snapshot id over all tag changes.
 	if sn.Original == nil {
 		sn.Original = sn.ID()

From ad14d6e4ac01a3ca3aa122e5aa72456e05a567db Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 9 Sep 2022 22:33:12 +0200
Subject: [PATCH 11/25] rewrite: use SelectByName like in the backup command

---
 cmd/restic/cmd_rewrite.go   | 8 ++++----
 internal/walker/rewriter.go | 8 +++++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 9e3ab370d..b4544719d 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -81,13 +81,13 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, err
 	}
 
-	checkExclude := func(nodepath string) bool {
+	selectByName := func(nodepath string) bool {
 		for _, reject := range rejectByNameFuncs {
 			if reject(nodepath) {
-				return true
+				return false
 			}
 		}
-		return false
+		return true
 	}
 
 	wg, wgCtx := errgroup.WithContext(ctx)
@@ -96,7 +96,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	var filteredTree restic.ID
 	wg.Go(func() error {
 		filteredTree, err = walker.FilterTree(wgCtx, repo, "/", *sn.Tree, &walker.TreeFilterVisitor{
-			CheckExclude: checkExclude,
+			SelectByName: selectByName,
 			PrintExclude: func(path string) { Verbosef(fmt.Sprintf("excluding %s\n", path)) },
 		})
 		if err != nil {
diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go
index e4e049da5..091c4bb2e 100644
--- a/internal/walker/rewriter.go
+++ b/internal/walker/rewriter.go
@@ -8,10 +8,12 @@ import (
 	"github.com/restic/restic/internal/restic"
 )
 
-type RejectByNameFunc func(path string) bool
+// SelectByNameFunc returns true for all items that should be included (files and
+// dirs). If false is returned, files are ignored and dirs are not even walked.
+type SelectByNameFunc func(item string) bool
 
 type TreeFilterVisitor struct {
-	CheckExclude RejectByNameFunc
+	SelectByName SelectByNameFunc
 	PrintExclude func(string)
 }
 
@@ -27,7 +29,7 @@ func FilterTree(ctx context.Context, repo restic.Repository, nodepath string, no
 	tb := restic.NewTreeJSONBuilder()
 	for _, node := range curTree.Nodes {
 		path := path.Join(nodepath, node.Name)
-		if visitor.CheckExclude(path) {
+		if !visitor.SelectByName(path) {
 			if visitor.PrintExclude != nil {
 				visitor.PrintExclude(path)
 			}

From 327f418a9cc6e5ea94f6fcb59bbc11218bf8dd81 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 9 Sep 2022 22:38:58 +0200
Subject: [PATCH 12/25] rewrite: cleanup err handling and output

---
 cmd/restic/cmd_rewrite.go | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index b4544719d..2f55e84ef 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -71,7 +71,7 @@ func init() {
 	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 
-func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions, gopts GlobalOptions) (bool, error) {
+func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
 	if sn.Tree == nil {
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
@@ -117,7 +117,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 	debug.Log("Snapshot %v modified", sn)
 	if opts.DryRun {
-		Printf("Would modify snapshot: %s\n", sn.String())
+		Verbosef("would save new snapshot\n")
 		return true, nil
 	}
 
@@ -145,7 +145,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 
 		debug.Log("old snapshot %v removed", sn.ID())
 	}
-	Printf("new snapshot saved as %v\n", id)
+	Verbosef("new snapshot saved as %v\n", id)
 	return true, nil
 }
 
@@ -183,24 +183,24 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 
 	changedCount := 0
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, opts.Hosts, opts.Tags, opts.Paths, args) {
-		Verbosef("Checking snapshot %s\n", sn.String())
-		changed, err := rewriteSnapshot(ctx, repo, sn, opts, gopts)
+		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		changed, err := rewriteSnapshot(ctx, repo, sn, opts)
 		if err != nil {
-			Warnf("unable to rewrite snapshot ID %q, ignoring: %v\n", sn.ID(), err)
-			continue
+			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
 		}
 		if changed {
 			changedCount++
 		}
 	}
 
+	Verbosef("\n")
 	if changedCount == 0 {
-		Verbosef("no snapshots modified\n")
+		Verbosef("no snapshots were modified\n")
 	} else {
 		if !opts.DryRun {
 			Verbosef("modified %v snapshots\n", changedCount)
 		} else {
-			Verbosef("dry run. would modify %v snapshots\n", changedCount)
+			Verbosef("would modify %v snapshots\n", changedCount)
 		}
 	}
 

From 375a3db64d3d8a1231fcda0a448176784cbe4ba3 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 9 Sep 2022 22:47:31 +0200
Subject: [PATCH 13/25] rewrite: non-exclusive lock if snapshots are only added

---
 cmd/restic/cmd_rewrite.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 2f55e84ef..7da90c674 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -161,9 +161,14 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 	}
 
 	if !opts.DryRun {
-		Verbosef("create exclusive lock for repository\n")
 		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo)
+		var err error
+		if opts.Inplace {
+			Verbosef("create exclusive lock for repository\n")
+			lock, ctx, err = lockRepoExclusive(ctx, repo)
+		} else {
+			lock, ctx, err = lockRepo(ctx, repo)
+		}
 		defer unlockRepo(lock)
 		if err != nil {
 			return err

From b0446491184d439dda805e6795ce0b8215c98e10 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 9 Sep 2022 23:33:18 +0200
Subject: [PATCH 14/25] rewrite: add minimal test

---
 cmd/restic/integration_rewrite_test.go | 38 ++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 cmd/restic/integration_rewrite_test.go

diff --git a/cmd/restic/integration_rewrite_test.go b/cmd/restic/integration_rewrite_test.go
new file mode 100644
index 000000000..b0f6b02ac
--- /dev/null
+++ b/cmd/restic/integration_rewrite_test.go
@@ -0,0 +1,38 @@
+package main
+
+import (
+	"context"
+	"path/filepath"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string) {
+	opts := RewriteOptions{
+		excludePatternOptions: excludePatternOptions{
+			Excludes: excludes,
+		},
+	}
+
+	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
+}
+
+func TestRewrite(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+
+	// create backup
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs)
+	testRunCheck(t, env.gopts)
+
+	// exclude some data
+	testRunRewriteExclude(t, env.gopts, []string{"3"})
+	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
+	testRunCheck(t, env.gopts)
+}

From a47d9a1c40a81aebdd232fd4fcff5c34d843095b Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 27 Sep 2022 21:19:32 +0200
Subject: [PATCH 15/25] rewrite: use unified snapshot filter options

---
 cmd/restic/cmd_rewrite.go | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 7da90c674..b52d4973c 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -47,12 +47,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
-	Hosts   []string
-	Paths   []string
-	Tags    restic.TagLists
 	Inplace bool
 	DryRun  bool
 
+	snapshotFilterOptions
 	excludePatternOptions
 }
 
@@ -62,12 +60,10 @@ func init() {
 	cmdRoot.AddCommand(cmdRewrite)
 
 	f := cmdRewrite.Flags()
-	f.StringArrayVarP(&rewriteOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
-	f.Var(&rewriteOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot-ID is given")
-	f.StringArrayVar(&rewriteOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot-ID is given")
 	f.BoolVarP(&rewriteOptions.Inplace, "inplace", "", false, "replace existing snapshots")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
 
+	initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true)
 	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 

From 73f54cc5ea72e70d4266cf20b71f23f353132cff Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Tue, 27 Sep 2022 21:21:14 +0200
Subject: [PATCH 16/25] rewrite: rename --inplace to --forget

---
 cmd/restic/cmd_rewrite.go | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index b52d4973c..a394b43d3 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -24,9 +24,9 @@ The "rewrite" command excludes files from existing snapshots.
 By default 'rewrite' will create new snapshots that will contains same data as
 the source snapshots but without excluded files. All metadata (time, host, tags)
 will be preserved. The special tag 'rewrite' will be added to new snapshots to
-distinguish it from the source (unless --inplace is used).
+distinguish it from the source (unless --forget is used).
 
-If --inplace option is used, old snapshot will be removed from repository.
+If --forget option is used, old snapshot will be removed from repository.
 
 Snapshots to rewrite are specified using --host, --tag, --path or by providing
 a list of snapshot ids. Not specifying a snapshot id will rewrite all snapshots.
@@ -47,8 +47,8 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 
 // RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
-	Inplace bool
-	DryRun  bool
+	Forget bool
+	DryRun bool
 
 	snapshotFilterOptions
 	excludePatternOptions
@@ -60,7 +60,7 @@ func init() {
 	cmdRoot.AddCommand(cmdRewrite)
 
 	f := cmdRewrite.Flags()
-	f.BoolVarP(&rewriteOptions.Inplace, "inplace", "", false, "replace existing snapshots")
+	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "replace existing snapshots")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
 
 	initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true)
@@ -123,7 +123,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	}
 	*sn.Tree = filteredTree
 
-	if !opts.Inplace {
+	if !opts.Forget {
 		sn.AddTags([]string{"rewrite"})
 	}
 
@@ -133,7 +133,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, err
 	}
 
-	if opts.Inplace {
+	if opts.Forget {
 		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
 		if err = repo.Backend().Remove(ctx, h); err != nil {
 			return false, err
@@ -159,7 +159,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 	if !opts.DryRun {
 		var lock *restic.Lock
 		var err error
-		if opts.Inplace {
+		if opts.Forget {
 			Verbosef("create exclusive lock for repository\n")
 			lock, ctx, err = lockRepoExclusive(ctx, repo)
 		} else {

From 0224e276ec597456b5e4fbe9852a7c4c55b363d4 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 14 Oct 2022 23:26:13 +0200
Subject: [PATCH 17/25] walker: Add tests for FilterTree

---
 internal/walker/rewriter.go      |   7 +-
 internal/walker/rewriter_test.go | 206 +++++++++++++++++++++++++++++++
 2 files changed, 212 insertions(+), 1 deletion(-)
 create mode 100644 internal/walker/rewriter_test.go

diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go
index 091c4bb2e..ece073f7b 100644
--- a/internal/walker/rewriter.go
+++ b/internal/walker/rewriter.go
@@ -17,7 +17,12 @@ type TreeFilterVisitor struct {
 	PrintExclude func(string)
 }
 
-func FilterTree(ctx context.Context, repo restic.Repository, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) {
+type BlobLoadSaver interface {
+	restic.BlobSaver
+	restic.BlobLoader
+}
+
+func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID restic.ID, visitor *TreeFilterVisitor) (newNodeID restic.ID, err error) {
 	curTree, err := restic.LoadTree(ctx, repo, nodeID)
 	if err != nil {
 		return restic.ID{}, err
diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go
new file mode 100644
index 000000000..e4ae33ec5
--- /dev/null
+++ b/internal/walker/rewriter_test.go
@@ -0,0 +1,206 @@
+package walker
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/pkg/errors"
+	"github.com/restic/restic/internal/restic"
+)
+
+// WritableTreeMap also support saving
+type WritableTreeMap struct {
+	TreeMap
+}
+
+func (t WritableTreeMap) SaveBlob(ctx context.Context, tpe restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) {
+	if tpe != restic.TreeBlob {
+		return restic.ID{}, false, 0, errors.New("can only save trees")
+	}
+
+	if id.IsNull() {
+		id = restic.Hash(buf)
+	}
+	_, ok := t.TreeMap[id]
+	if ok {
+		return id, false, 0, nil
+	}
+
+	t.TreeMap[id] = append([]byte{}, buf...)
+	return id, true, len(buf), nil
+}
+
+func (t WritableTreeMap) Dump() {
+	for k, v := range t.TreeMap {
+		fmt.Printf("%v: %v", k, string(v))
+	}
+}
+
+type checkRewriteFunc func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB))
+
+// checkRewriteItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'.
+func checkRewriteItemOrder(want []string) checkRewriteFunc {
+	pos := 0
+	return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) {
+		vis := TreeFilterVisitor{
+			SelectByName: func(path string) bool {
+				if pos >= len(want) {
+					t.Errorf("additional unexpected path found: %v", path)
+					return false
+				}
+
+				if path != want[pos] {
+					t.Errorf("wrong path found, want %q, got %q", want[pos], path)
+				}
+				pos++
+				return true
+			},
+		}
+
+		final = func(t testing.TB) {
+			if pos != len(want) {
+				t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
+			}
+		}
+
+		return vis, final
+	}
+}
+
+// checkRewriteSkips excludes nodes if path is in skipFor, it checks that all excluded entries are printed.
+func checkRewriteSkips(skipFor map[string]struct{}, want []string) checkRewriteFunc {
+	var pos int
+	printed := make(map[string]struct{})
+
+	return func(t testing.TB) (visitor TreeFilterVisitor, final func(testing.TB)) {
+		vis := TreeFilterVisitor{
+			SelectByName: func(path string) bool {
+				if pos >= len(want) {
+					t.Errorf("additional unexpected path found: %v", path)
+					return false
+				}
+
+				if path != want[pos] {
+					t.Errorf("wrong path found, want %q, got %q", want[pos], path)
+				}
+				pos++
+
+				_, ok := skipFor[path]
+				return !ok
+			},
+			PrintExclude: func(s string) {
+				if _, ok := printed[s]; ok {
+					t.Errorf("path was already printed %v", s)
+				}
+				printed[s] = struct{}{}
+			},
+		}
+
+		final = func(t testing.TB) {
+			if !cmp.Equal(skipFor, printed) {
+				t.Errorf("unexpected paths skipped: %s", cmp.Diff(skipFor, printed))
+			}
+			if pos != len(want) {
+				t.Errorf("not enough items returned, want %d, got %d", len(want), pos)
+			}
+		}
+
+		return vis, final
+	}
+}
+
+func TestRewriter(t *testing.T) {
+	var tests = []struct {
+		tree    TestTree
+		newTree TestTree
+		check   checkRewriteFunc
+	}{
+		{ // don't change
+			tree: TestTree{
+				"foo": TestFile{},
+				"subdir": TestTree{
+					"subfile": TestFile{},
+				},
+			},
+			check: checkRewriteItemOrder([]string{
+				"/foo",
+				"/subdir",
+				"/subdir/subfile",
+			}),
+		},
+		{ // exclude file
+			tree: TestTree{
+				"foo": TestFile{},
+				"subdir": TestTree{
+					"subfile": TestFile{},
+				},
+			},
+			newTree: TestTree{
+				"foo":    TestFile{},
+				"subdir": TestTree{},
+			},
+			check: checkRewriteSkips(
+				map[string]struct{}{
+					"/subdir/subfile": {},
+				},
+				[]string{
+					"/foo",
+					"/subdir",
+					"/subdir/subfile",
+				},
+			),
+		},
+		{ // exclude dir
+			tree: TestTree{
+				"foo": TestFile{},
+				"subdir": TestTree{
+					"subfile": TestFile{},
+				},
+			},
+			newTree: TestTree{
+				"foo": TestFile{},
+			},
+			check: checkRewriteSkips(
+				map[string]struct{}{
+					"/subdir": {},
+				},
+				[]string{
+					"/foo",
+					"/subdir",
+				},
+			),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run("", func(t *testing.T) {
+			repo, root := BuildTreeMap(test.tree)
+			if test.newTree == nil {
+				test.newTree = test.tree
+			}
+			expRepo, expRoot := BuildTreeMap(test.newTree)
+			modrepo := WritableTreeMap{repo}
+
+			ctx, cancel := context.WithCancel(context.TODO())
+			defer cancel()
+
+			vis, last := test.check(t)
+			newRoot, err := FilterTree(ctx, modrepo, "/", root, &vis)
+			if err != nil {
+				t.Error(err)
+			}
+			last(t)
+
+			// verifying against the expected tree root also implicitly checks the structural integrity
+			if newRoot != expRoot {
+				t.Error("hash mismatch")
+				fmt.Println("Got")
+				modrepo.Dump()
+				fmt.Println("Expected")
+				WritableTreeMap{expRepo}.Dump()
+			}
+		})
+	}
+}

From ec0c91e2333d304053665ef62c7a8c85825cbc2a Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Fri, 14 Oct 2022 23:44:10 +0200
Subject: [PATCH 18/25] rewrite: Add tests for further ways to use the command

---
 cmd/restic/integration_rewrite_test.go | 49 ++++++++++++++++++++++----
 1 file changed, 42 insertions(+), 7 deletions(-)

diff --git a/cmd/restic/integration_rewrite_test.go b/cmd/restic/integration_rewrite_test.go
index b0f6b02ac..e6007973b 100644
--- a/cmd/restic/integration_rewrite_test.go
+++ b/cmd/restic/integration_rewrite_test.go
@@ -5,23 +5,22 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string) {
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
 	opts := RewriteOptions{
 		excludePatternOptions: excludePatternOptions{
 			Excludes: excludes,
 		},
+		Forget: forget,
 	}
 
 	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
 }
 
-func TestRewrite(t *testing.T) {
-	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
+func createBasicRewriteRepo(t testing.TB, env *testEnvironment) restic.ID {
 	testSetupBackupData(t, env)
 
 	// create backup
@@ -30,9 +29,45 @@ func TestRewrite(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 1, "expected one snapshot, got %v", snapshotIDs)
 	testRunCheck(t, env.gopts)
 
+	return snapshotIDs[0]
+}
+
+func TestRewrite(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"})
-	snapshotIDs = testRunList(t, "snapshots", env.gopts)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+	snapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
 	testRunCheck(t, env.gopts)
 }
+
+func TestRewriteUnchanged(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	snapshotID := createBasicRewriteRepo(t, env)
+
+	// use an exclude that will not exclude anything
+	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
+	testRunCheck(t, env.gopts)
+}
+
+func TestRewriteReplace(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	snapshotID := createBasicRewriteRepo(t, env)
+
+	// exclude some data
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
+	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
+	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
+	// check forbids unused blobs, thus remove them first
+	testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
+	testRunCheck(t, env.gopts)
+}

From 11b8c3a1582b2222883ffacfaebb696152f208d0 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 15 Oct 2022 00:24:16 +0200
Subject: [PATCH 19/25] rewrite: add documentation

---
 doc/040_backup.rst             |  1 +
 doc/045_working_with_repos.rst | 51 ++++++++++++++++++++++++++++++++++
 doc/manual_rest.rst            |  3 +-
 3 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/doc/040_backup.rst b/doc/040_backup.rst
index c1cd940c7..b9996311d 100644
--- a/doc/040_backup.rst
+++ b/doc/040_backup.rst
@@ -204,6 +204,7 @@ Combined with ``--verbose``, you can see a list of changes:
     modified  /archive.tar.gz, saved in 0.140s (25.542 MiB added)
     Would be added to the repository: 25.551 MiB
 
+.. _backup-excluding-files:
 Excluding Files
 ***************
 
diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst
index 6da4b707b..a42dbab9e 100644
--- a/doc/045_working_with_repos.rst
+++ b/doc/045_working_with_repos.rst
@@ -136,6 +136,7 @@ or the environment variable ``$RESTIC_FROM_KEY_HINT``.
     repository. You can avoid this limitation by using the rclone backend
     along with remotes which are configured in rclone.
 
+.. _copy-filtering-snapshots:
 Filtering snapshots to copy
 ---------------------------
 
@@ -175,6 +176,56 @@ using the same chunker parameters as the source repository:
 Note that it is not possible to change the chunker parameters of an existing repository.
 
 
+Removing files from snapshots
+=============================
+
+Sometimes a backup includes more files that intended. Instead of removing the snapshot,
+it is possible to rewrite its contents to remove the files in question. For this you
+can use the ``rewrite`` command:
+
+.. code-block:: console
+
+    $ restic -r /srv/restic-repo rewrite --exclude secret-file
+    repository c881945a opened (repository version 2) successfully, password is correct
+
+    snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
+    excluding /home/user/work/secret-file
+    new snapshot saved as b6aee1ff7f5e0ac15157f16370015978e496fa60f7351bc94a8d6049e4c7096d
+
+    snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST)
+
+    modified 1 snapshots
+
+    $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2
+    repository c881945a opened (repository version 2) successfully, password is correct
+
+    snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
+    excluding /home/user/work/secret-file
+    new snapshot saved as b6aee1ff7f5e0ac15157f16370015978e496fa60f7351bc94a8d6049e4c7096d
+
+    modified 1 snapshots
+
+The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and ``--iexclude-file`` are
+supported. They behave the same way as for the backup command, see :ref:`backup-excluding-files`
+for details.
+
+It is possible to only rewrite a subset of snapshots. Filtering the snapshots works the
+same way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
+
+By default, the ``rewrite`` command will keep the original snapshot and create a new
+snapshot for every snapshot which was modified while rewriting. All new snapshots are
+marked with the tag ``rewrite``.
+
+Alternatively, you can use the ``--forget`` option to immediatelly remove the original
+snapshot. In this case, no tag is added to the snapshots. Please note that only the
+original snapshot file is removed from the repository, but not the excluded data.
+Run the ``prune`` command afterwards to cleanup the now unused data.
+
+In order to preview the changes which ``rewrite`` would make, you can use the ``--dry-run``
+option. This will simulate the rewriting process without actually modifying the repository.
+Instead restic will only print the expected changes.
+
+
 Checking integrity and consistency
 ==================================
 
diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst
index f2d090209..a9c08b233 100644
--- a/doc/manual_rest.rst
+++ b/doc/manual_rest.rst
@@ -26,7 +26,7 @@ Usage help is available:
       dump          Print a backed-up file to stdout
       find          Find a file, a directory or restic IDs
       forget        Remove snapshots from the repository
-      generate      Generate manual pages and auto-completion files (bash, fish, zsh)
+      generate      Generate manual pages and auto-completion files (bash, fish, zsh, powershell)
       help          Help about any command
       init          Initialize a new repository
       key           Manage keys (passwords)
@@ -38,6 +38,7 @@ Usage help is available:
       rebuild-index Build a new index
       recover       Recover data from the repository not referenced by snapshots
       restore       Extract the data from a snapshot
+      rewrite       Rewrite existing snapshots
       self-update   Update the restic binary
       snapshots     List all snapshots
       stats         Scan the repository and show basic statistics

From f88acd45039443ad18b6b415e7b8fde2c59fcfe3 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 15 Oct 2022 10:14:50 +0200
Subject: [PATCH 20/25] rewrite: Fail if a tree contains an unknown field

In principle, the JSON format of Tree objects is extensible without
requiring a format change. In order to not loose information just play
it safe and reject rewriting trees for which we could loose data.
---
 internal/walker/rewriter.go      | 12 ++++++++++++
 internal/walker/rewriter_test.go | 16 ++++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go
index ece073f7b..6f063831e 100644
--- a/internal/walker/rewriter.go
+++ b/internal/walker/rewriter.go
@@ -2,6 +2,7 @@ package walker
 
 import (
 	"context"
+	"fmt"
 	"path"
 
 	"github.com/restic/restic/internal/debug"
@@ -28,6 +29,17 @@ func FilterTree(ctx context.Context, repo BlobLoadSaver, nodepath string, nodeID
 		return restic.ID{}, err
 	}
 
+	// check that we can properly encode this tree without losing information
+	// The alternative of using json/Decoder.DisallowUnknownFields() doesn't work as we use
+	// a custom UnmarshalJSON to decode trees, see also https://github.com/golang/go/issues/41144
+	testID, err := restic.SaveTree(ctx, repo, curTree)
+	if err != nil {
+		return restic.ID{}, err
+	}
+	if nodeID != testID {
+		return restic.ID{}, fmt.Errorf("cannot encode tree at %q without loosing information", nodepath)
+	}
+
 	debug.Log("filterTree: %s, nodeId: %s\n", nodepath, nodeID.Str())
 
 	changed := false
diff --git a/internal/walker/rewriter_test.go b/internal/walker/rewriter_test.go
index e4ae33ec5..3dcf0ac9e 100644
--- a/internal/walker/rewriter_test.go
+++ b/internal/walker/rewriter_test.go
@@ -204,3 +204,19 @@ func TestRewriter(t *testing.T) {
 		})
 	}
 }
+
+func TestRewriterFailOnUnknownFields(t *testing.T) {
+	tm := WritableTreeMap{TreeMap{}}
+	node := []byte(`{"nodes":[{"name":"subfile","type":"file","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","uid":0,"gid":0,"content":null,"unknown_field":42}]}`)
+	id := restic.Hash(node)
+	tm.TreeMap[id] = node
+
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+	// use nil visitor to crash if the tree loading works unexpectedly
+	_, err := FilterTree(ctx, tm, "/", id, nil)
+
+	if err == nil {
+		t.Error("missing error on unknown field")
+	}
+}

From c15bedccc003702d71a45400f9b769a2d30cbb08 Mon Sep 17 00:00:00 2001
From: "Leo R. Lundgren" <leo@finalresort.org>
Date: Tue, 25 Oct 2022 00:54:28 +0200
Subject: [PATCH 21/25] rewrite: Revert unrelated documentation change

---
 doc/manual_rest.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst
index a9c08b233..b15e1dd69 100644
--- a/doc/manual_rest.rst
+++ b/doc/manual_rest.rst
@@ -26,7 +26,7 @@ Usage help is available:
       dump          Print a backed-up file to stdout
       find          Find a file, a directory or restic IDs
       forget        Remove snapshots from the repository
-      generate      Generate manual pages and auto-completion files (bash, fish, zsh, powershell)
+      generate      Generate manual pages and auto-completion files (bash, fish, zsh)
       help          Help about any command
       init          Initialize a new repository
       key           Manage keys (passwords)

From f86ef4d3ddbd720694581f5215e49a507e9ec10f Mon Sep 17 00:00:00 2001
From: "Leo R. Lundgren" <leo@finalresort.org>
Date: Tue, 25 Oct 2022 00:59:18 +0200
Subject: [PATCH 22/25] rewrite: Polish code and add missing messages

---
 cmd/restic/cmd_rewrite.go | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index a394b43d3..688ee77a2 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -114,6 +114,11 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 	debug.Log("Snapshot %v modified", sn)
 	if opts.DryRun {
 		Verbosef("would save new snapshot\n")
+
+		if opts.Forget {
+			Verbosef("would remove old snapshot\n")
+		}
+
 		return true, nil
 	}
 
@@ -138,15 +143,14 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		if err = repo.Backend().Remove(ctx, h); err != nil {
 			return false, err
 		}
-
-		debug.Log("old snapshot %v removed", sn.ID())
+		debug.Log("removed old snapshot %v", sn.ID())
+		Verbosef("removed old snapshot %v\n", sn.ID().Str())
 	}
-	Verbosef("new snapshot saved as %v\n", id)
+	Verbosef("saved new snapshot %v\n", id.Str())
 	return true, nil
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-
 	if len(opts.ExcludeFiles) == 0 && len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 {
 		return errors.Fatal("Nothing to do: no excludes provided")
 	}
@@ -196,7 +200,11 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 
 	Verbosef("\n")
 	if changedCount == 0 {
-		Verbosef("no snapshots were modified\n")
+		if !opts.DryRun {
+			Verbosef("no snapshots were modified\n")
+		} else {
+			Verbosef("no snapshots would be modified\n")
+		}
 	} else {
 		if !opts.DryRun {
 			Verbosef("modified %v snapshots\n", changedCount)

From f175da2756e0a97de1322145dcb67841efde513e Mon Sep 17 00:00:00 2001
From: "Leo R. Lundgren" <leo@finalresort.org>
Date: Tue, 25 Oct 2022 01:01:47 +0200
Subject: [PATCH 23/25] rewrite: Polish documentation

---
 cmd/restic/cmd_rewrite.go      | 27 +++++++++----------
 doc/045_working_with_repos.rst | 47 +++++++++++++++++++---------------
 doc/manual_rest.rst            |  2 +-
 3 files changed, 41 insertions(+), 35 deletions(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index 688ee77a2..fc6284ce8 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -17,22 +17,23 @@ import (
 
 var cmdRewrite = &cobra.Command{
 	Use:   "rewrite [flags] [snapshotID ...]",
-	Short: "Rewrite existing snapshots",
+	Short: "Rewrite snapshots to exclude unwanted files",
 	Long: `
-The "rewrite" command excludes files from existing snapshots.
+The "rewrite" command excludes files from existing snapshots. It creates new
+snapshots containing the same data as the original ones, but without the files
+you specify to exclude. All metadata (time, host, tags) will be preserved.
 
-By default 'rewrite' will create new snapshots that will contains same data as
-the source snapshots but without excluded files. All metadata (time, host, tags)
-will be preserved. The special tag 'rewrite' will be added to new snapshots to
-distinguish it from the source (unless --forget is used).
+The snapshots to rewrite are specified using the --host, --tag and --path options,
+or by providing a list of snapshot IDs. Please note that specifying neither any of
+these options nor a snapshot ID will cause the command to rewrite all snapshots.
 
-If --forget option is used, old snapshot will be removed from repository.
+The special tag 'rewrite' will be added to the new snapshots to distinguish
+them from the original ones, unless --forget is used. If the --forget option is
+used, the original snapshots will instead be directly removed from the repository.
 
-Snapshots to rewrite are specified using --host, --tag, --path or by providing
-a list of snapshot ids. Not specifying a snapshot id will rewrite all snapshots.
-
-Please note, that this command only creates new snapshots. In order to delete
-data from the repository use 'prune' command.
+Please note that the --forget option only removes the snapshots and not the actual
+data stored in the repository. In order to delete the no longer referenced data,
+use the "prune" command.
 
 EXIT STATUS
 ===========
@@ -60,7 +61,7 @@ func init() {
 	cmdRoot.AddCommand(cmdRewrite)
 
 	f := cmdRewrite.Flags()
-	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "replace existing snapshots")
+	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
 
 	initMultiSnapshotFilterOptions(f, &rewriteOptions.snapshotFilterOptions, true)
diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst
index a42dbab9e..860d01ae3 100644
--- a/doc/045_working_with_repos.rst
+++ b/doc/045_working_with_repos.rst
@@ -141,7 +141,7 @@ Filtering snapshots to copy
 ---------------------------
 
 The list of snapshots to copy can be filtered by host, path in the backup
-and / or a comma-separated tag list:
+and/or a comma-separated tag list:
 
 .. code-block:: console
 
@@ -179,9 +179,11 @@ Note that it is not possible to change the chunker parameters of an existing rep
 Removing files from snapshots
 =============================
 
-Sometimes a backup includes more files that intended. Instead of removing the snapshot,
-it is possible to rewrite its contents to remove the files in question. For this you
-can use the ``rewrite`` command:
+Snapshots sometimes turn out to include more files that intended. Instead of
+removing the snapshots entirely and running the corresponding backup commands
+again (which is not always practical after the fact) it is possible to remove
+the unwanted files from affected snapshots by rewriting them using the
+``rewrite`` command:
 
 .. code-block:: console
 
@@ -190,7 +192,7 @@ can use the ``rewrite`` command:
 
     snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
     excluding /home/user/work/secret-file
-    new snapshot saved as b6aee1ff7f5e0ac15157f16370015978e496fa60f7351bc94a8d6049e4c7096d
+    saved new snapshot b6aee1ff
 
     snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST)
 
@@ -201,29 +203,32 @@ can use the ``rewrite`` command:
 
     snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST)
     excluding /home/user/work/secret-file
-    new snapshot saved as b6aee1ff7f5e0ac15157f16370015978e496fa60f7351bc94a8d6049e4c7096d
+    new snapshot saved as b6aee1ff
 
     modified 1 snapshots
 
-The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and ``--iexclude-file`` are
-supported. They behave the same way as for the backup command, see :ref:`backup-excluding-files`
-for details.
+The options ``--exclude``, ``--exclude-file``, ``--iexclude`` and
+``--iexclude-file`` are supported. They behave the same way as for the backup
+command, see :ref:`backup-excluding-files` for details.
 
-It is possible to only rewrite a subset of snapshots. Filtering the snapshots works the
-same way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
+It is possible to rewrite only a subset of snapshots by filtering them the same
+way as for the ``copy`` command, see :ref:`copy-filtering-snapshots`.
 
-By default, the ``rewrite`` command will keep the original snapshot and create a new
-snapshot for every snapshot which was modified while rewriting. All new snapshots are
-marked with the tag ``rewrite``.
+By default, the ``rewrite`` command will keep the original snapshots and create
+new ones for every snapshot which was modified during rewriting. The new
+snapshots are marked with the tag ``rewrite`` to differentiate them from the
+original, rewritten snapshots.
 
-Alternatively, you can use the ``--forget`` option to immediatelly remove the original
-snapshot. In this case, no tag is added to the snapshots. Please note that only the
-original snapshot file is removed from the repository, but not the excluded data.
-Run the ``prune`` command afterwards to cleanup the now unused data.
+Alternatively, you can use the ``--forget`` option to immediately remove the
+original snapshots. In this case, no tag is added to the new snapshots. Please
+note that this only removes the snapshots and not the actual data stored in the
+repository. Run the ``prune`` command afterwards to remove the now unreferenced
+data (just like when having used the ``forget`` command).
 
-In order to preview the changes which ``rewrite`` would make, you can use the ``--dry-run``
-option. This will simulate the rewriting process without actually modifying the repository.
-Instead restic will only print the expected changes.
+In order to preview the changes which ``rewrite`` would make, you can use the
+``--dry-run`` option. This will simulate the rewriting process without actually
+modifying the repository. Instead restic will only print the actions it would
+perform.
 
 
 Checking integrity and consistency
diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst
index b15e1dd69..1aa9a434d 100644
--- a/doc/manual_rest.rst
+++ b/doc/manual_rest.rst
@@ -38,7 +38,7 @@ Usage help is available:
       rebuild-index Build a new index
       recover       Recover data from the repository not referenced by snapshots
       restore       Extract the data from a snapshot
-      rewrite       Rewrite existing snapshots
+      rewrite       Rewrite snapshots to exclude unwanted files
       self-update   Update the restic binary
       snapshots     List all snapshots
       stats         Scan the repository and show basic statistics

From 537cfe2e4c61a86790ff2876f1838e0910d39cba Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 12 Nov 2022 19:50:59 +0100
Subject: [PATCH 24/25] rewrite: Fix check that an exclude pattern was passed

The old check did not consider files containing case insensitive
excludes. The check is now implemented as a function of the
excludePatternOptions struct to improve cohesion.
---
 cmd/restic/cmd_rewrite.go | 2 +-
 cmd/restic/exclude.go     | 4 ++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index fc6284ce8..abab5dfcd 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -152,7 +152,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if len(opts.ExcludeFiles) == 0 && len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 {
+	if opts.excludePatternOptions.Empty() {
 		return errors.Fatal("Nothing to do: no excludes provided")
 	}
 
diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go
index 86f85f133..4a1954b88 100644
--- a/cmd/restic/exclude.go
+++ b/cmd/restic/exclude.go
@@ -475,6 +475,10 @@ func initExcludePatternOptions(f *pflag.FlagSet, opts *excludePatternOptions) {
 	f.StringArrayVar(&opts.InsensitiveExcludeFiles, "iexclude-file", nil, "same as --exclude-file but ignores casing of `file`names in patterns")
 }
 
+func (opts *excludePatternOptions) Empty() bool {
+	return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
+}
+
 func collectExcludePatterns(opts excludePatternOptions) ([]RejectByNameFunc, error) {
 	var fs []RejectByNameFunc
 	// add patterns from file

From bb0fa76c0641f1e9bb7a4684575dbe1028fd5893 Mon Sep 17 00:00:00 2001
From: Michael Eischer <michael.eischer@fau.de>
Date: Sat, 12 Nov 2022 19:54:52 +0100
Subject: [PATCH 25/25] Cleanup exclude pattern collection

---
 cmd/restic/cmd_backup.go  | 2 +-
 cmd/restic/cmd_rewrite.go | 2 +-
 cmd/restic/exclude.go     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go
index 8b1f13b55..2b64217c2 100644
--- a/cmd/restic/cmd_backup.go
+++ b/cmd/restic/cmd_backup.go
@@ -306,7 +306,7 @@ func collectRejectByNameFuncs(opts BackupOptions, repo *repository.Repository, t
 		fs = append(fs, f)
 	}
 
-	fsPatterns, err := collectExcludePatterns(opts.excludePatternOptions)
+	fsPatterns, err := opts.excludePatternOptions.CollectPatterns()
 	if err != nil {
 		return nil, err
 	}
diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go
index abab5dfcd..2a750b969 100644
--- a/cmd/restic/cmd_rewrite.go
+++ b/cmd/restic/cmd_rewrite.go
@@ -73,7 +73,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
 	}
 
-	rejectByNameFuncs, err := collectExcludePatterns(opts.excludePatternOptions)
+	rejectByNameFuncs, err := opts.excludePatternOptions.CollectPatterns()
 	if err != nil {
 		return false, err
 	}
diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go
index 4a1954b88..efe6f41e4 100644
--- a/cmd/restic/exclude.go
+++ b/cmd/restic/exclude.go
@@ -479,7 +479,7 @@ func (opts *excludePatternOptions) Empty() bool {
 	return len(opts.Excludes) == 0 && len(opts.InsensitiveExcludes) == 0 && len(opts.ExcludeFiles) == 0 && len(opts.InsensitiveExcludeFiles) == 0
 }
 
-func collectExcludePatterns(opts excludePatternOptions) ([]RejectByNameFunc, error) {
+func (opts excludePatternOptions) CollectPatterns() ([]RejectByNameFunc, error) {
 	var fs []RejectByNameFunc
 	// add patterns from file
 	if len(opts.ExcludeFiles) > 0 {