From 0256f95994fabc670f57e6b4dfa4ea8a1c9bf417 Mon Sep 17 00:00:00 2001 From: DRON-666 <64691982+DRON-666@users.noreply.github.com> Date: Tue, 10 Nov 2020 01:22:27 +0300 Subject: [PATCH] dump: Split tar and walk logic --- cmd/restic/cmd_dump.go | 17 ++++--- internal/dump/common.go | 104 ++++++++++++++++++++++++++++++++++++++++ internal/dump/tar.go | 101 ++++++-------------------------------- 3 files changed, 127 insertions(+), 95 deletions(-) create mode 100644 internal/dump/common.go diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index d3ca87923..7875baa89 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -65,8 +65,7 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string) error { - +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, writeDump dump.WriteDump) error { if tree == nil { return fmt.Errorf("called with a nil tree") } @@ -81,10 +80,10 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { - if err := checkStdoutTar(); err != nil { + if err := checkStdoutArchive(); err != nil { return err } - return dump.WriteTar(ctx, repo, tree, "/", os.Stdout) + return writeDump(ctx, repo, tree, "/", os.Stdout) } item := filepath.Join(prefix, pathComponents[0]) @@ -100,16 +99,16 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:]) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], writeDump) case dump.IsDir(node): - if err := checkStdoutTar(); err != nil { + if err := checkStdoutArchive(); err != nil { return err } subtree, err := repo.LoadTree(ctx, *node.Subtree) if err != nil { return err } - return dump.WriteTar(ctx, repo, subtree, item, os.Stdout) + return writeDump(ctx, repo, subtree, item, os.Stdout) case l > 1: return fmt.Errorf("%q should be a dir, but is a %q", item, node.Type) case !dump.IsFile(node): @@ -176,7 +175,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { Exitf(2, "loading tree for snapshot %q failed: %v", snapshotIDString, err) } - err = printFromTree(ctx, tree, repo, "/", splittedPath) + err = printFromTree(ctx, tree, repo, "/", splittedPath, dump.WriteTar) if err != nil { Exitf(2, "cannot dump file: %v", err) } @@ -184,7 +183,7 @@ func runDump(opts DumpOptions, gopts GlobalOptions, args []string) error { return nil } -func checkStdoutTar() error { +func checkStdoutArchive() error { if stdoutIsTerminal() { return fmt.Errorf("stdout is the terminal, please redirect output") } diff --git a/internal/dump/common.go b/internal/dump/common.go new file mode 100644 index 000000000..5b3d82068 --- /dev/null +++ b/internal/dump/common.go @@ -0,0 +1,104 @@ +package dump + +import ( + "context" + "io" + "path" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/walker" +) + +// dumper implements saving node data. +type dumper interface { + dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error +} + +// WriteDump will write the contents of the given tree to the given destination. +// It will loop over all nodes in the tree and dump them recursively. +type WriteDump func(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error + +func writeDump(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dmp dumper, dst io.Writer) error { + for _, rootNode := range tree.Nodes { + rootNode.Path = rootPath + err := dumpTree(ctx, repo, rootNode, rootPath, dmp) + if err != nil { + return err + } + } + + return nil +} + +func dumpTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node, rootPath string, dmp dumper) error { + rootNode.Path = path.Join(rootNode.Path, rootNode.Name) + rootPath = rootNode.Path + + if err := dmp.dumpNode(ctx, rootNode, repo); err != nil { + return err + } + + // If this is no directory we are finished + if !IsDir(rootNode) { + return nil + } + + err := walker.Walk(ctx, repo, *rootNode.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + if err != nil { + return false, err + } + if node == nil { + return false, nil + } + + node.Path = path.Join(rootPath, nodepath) + + if IsFile(node) || IsLink(node) || IsDir(node) { + err := dmp.dumpNode(ctx, node, repo) + if err != nil { + return false, err + } + } + + return false, nil + }) + + return err +} + +// GetNodeData will write the contents of the node to the given output. +func GetNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error { + var ( + buf []byte + err error + ) + for _, id := range node.Content { + buf, err = repo.LoadBlob(ctx, restic.DataBlob, id, buf) + if err != nil { + return err + } + + _, err = output.Write(buf) + if err != nil { + return errors.Wrap(err, "Write") + } + } + + return nil +} + +// IsDir checks if the given node is a directory. +func IsDir(node *restic.Node) bool { + return node.Type == "dir" +} + +// IsLink checks if the given node as a link. +func IsLink(node *restic.Node) bool { + return node.Type == "symlink" +} + +// IsFile checks if the given node is a file. +func IsFile(node *restic.Node) bool { + return node.Type == "file" +} diff --git a/internal/dump/tar.go b/internal/dump/tar.go index 816a0ffcf..02fa8e185 100644 --- a/internal/dump/tar.go +++ b/internal/dump/tar.go @@ -5,65 +5,31 @@ import ( "context" "io" "os" - "path" "path/filepath" "strings" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/walker" ) -// WriteTar will write the contents of the given tree, encoded as a tar to the given destination. -// It will loop over all nodes in the tree and dump them recursively. -func WriteTar(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error { - tw := tar.NewWriter(dst) - - for _, rootNode := range tree.Nodes { - rootNode.Path = rootPath - err := tarTree(ctx, repo, rootNode, rootPath, tw) - if err != nil { - _ = tw.Close() - return err - } - } - return tw.Close() +type tarDumper struct { + w *tar.Writer } -func tarTree(ctx context.Context, repo restic.Repository, rootNode *restic.Node, rootPath string, tw *tar.Writer) error { - rootNode.Path = path.Join(rootNode.Path, rootNode.Name) - rootPath = rootNode.Path +// Statically ensure that tarDumper implements dumper. +var _ dumper = tarDumper{} - if err := tarNode(ctx, tw, rootNode, repo); err != nil { +// WriteTar will write the contents of the given tree, encoded as a tar to the given destination. +func WriteTar(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error { + dmp := tarDumper{w: tar.NewWriter(dst)} + + err := writeDump(ctx, repo, tree, rootPath, dmp, dst) + if err != nil { + dmp.w.Close() return err } - // If this is no directory we are finished - if !IsDir(rootNode) { - return nil - } - - err := walker.Walk(ctx, repo, *rootNode.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { - if err != nil { - return false, err - } - if node == nil { - return false, nil - } - - node.Path = path.Join(rootPath, nodepath) - - if IsFile(node) || IsLink(node) || IsDir(node) { - err := tarNode(ctx, tw, node, repo) - if err != nil { - return false, err - } - } - - return false, nil - }) - - return err + return dmp.w.Close() } // copied from archive/tar.FileInfoHeader @@ -75,7 +41,7 @@ const ( c_ISVTX = 01000 // Save text (sticky bit) ) -func tarNode(ctx context.Context, tw *tar.Writer, node *restic.Node, repo restic.Repository) error { +func (dmp tarDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error { relPath, err := filepath.Rel("/", node.Path) if err != nil { return err @@ -120,13 +86,13 @@ func tarNode(ctx context.Context, tw *tar.Writer, node *restic.Node, repo restic header.Name += "/" } - err = tw.WriteHeader(header) + err = dmp.w.WriteHeader(header) if err != nil { return errors.Wrap(err, "TarHeader ") } - return GetNodeData(ctx, tw, repo, node) + return GetNodeData(ctx, dmp.w, repo, node) } func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { @@ -146,7 +112,6 @@ func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { tmpMap["SCHILY.acl.default"] = na.String() } } - } else { tmpMap["SCHILY.xattr."+attr.Name] = attrString } @@ -154,39 +119,3 @@ func parseXattrs(xattrs []restic.ExtendedAttribute) map[string]string { return tmpMap } - -// GetNodeData will write the contents of the node to the given output -func GetNodeData(ctx context.Context, output io.Writer, repo restic.Repository, node *restic.Node) error { - var ( - buf []byte - err error - ) - for _, id := range node.Content { - buf, err = repo.LoadBlob(ctx, restic.DataBlob, id, buf) - if err != nil { - return err - } - - _, err = output.Write(buf) - if err != nil { - return errors.Wrap(err, "Write") - } - - } - return nil -} - -// IsDir checks if the given node is a directory -func IsDir(node *restic.Node) bool { - return node.Type == "dir" -} - -// IsLink checks if the given node as a link -func IsLink(node *restic.Node) bool { - return node.Type == "symlink" -} - -// IsFile checks if the given node is a file -func IsFile(node *restic.Node) bool { - return node.Type == "file" -}