restic/node.go

455 lines
9.6 KiB
Go
Raw Normal View History

2015-02-03 20:07:55 +00:00
package restic
import (
"encoding/json"
"fmt"
"os"
2015-04-29 03:45:24 +00:00
"os/user"
2015-02-03 20:07:55 +00:00
"strconv"
2015-05-01 21:26:40 +00:00
"sync"
2015-02-03 20:07:55 +00:00
"syscall"
"time"
"github.com/juju/errors"
2015-02-03 20:07:55 +00:00
"github.com/restic/restic/backend"
2015-04-29 03:31:07 +00:00
"github.com/restic/restic/debug"
"github.com/restic/restic/pack"
"github.com/restic/restic/repository"
2015-02-03 20:07:55 +00:00
)
2015-05-02 14:12:19 +00:00
// Node is a file, directory or other item in a backup.
2015-02-03 20:07:55 +00:00
type Node struct {
Name string `json:"name"`
Type string `json:"type"`
Mode os.FileMode `json:"mode,omitempty"`
ModTime time.Time `json:"mtime,omitempty"`
AccessTime time.Time `json:"atime,omitempty"`
ChangeTime time.Time `json:"ctime,omitempty"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
Inode uint64 `json:"inode,omitempty"`
Size uint64 `json:"size,omitempty"`
Links uint64 `json:"links,omitempty"`
LinkTarget string `json:"linktarget,omitempty"`
Device uint64 `json:"device,omitempty"`
Content []backend.ID `json:"content"`
Subtree backend.ID `json:"subtree,omitempty"`
Error string `json:"error,omitempty"`
tree *Tree
2015-02-15 13:44:54 +00:00
path string
err error
blobs repository.Blobs
2015-02-03 20:07:55 +00:00
}
2015-04-28 06:11:01 +00:00
func (node Node) String() string {
switch node.Type {
2015-02-03 20:07:55 +00:00
case "file":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
2015-04-28 06:11:01 +00:00
node.Mode, node.UID, node.GID, node.Size, node.ModTime, node.Name)
2015-02-03 20:07:55 +00:00
case "dir":
return fmt.Sprintf("%s %5d %5d %6d %s %s",
2015-04-28 06:11:01 +00:00
node.Mode|os.ModeDir, node.UID, node.GID, node.Size, node.ModTime, node.Name)
2015-02-03 20:07:55 +00:00
}
2015-04-28 06:11:01 +00:00
return fmt.Sprintf("<Node(%s) %s>", node.Type, node.Name)
2015-02-03 20:07:55 +00:00
}
func (node Node) Tree() *Tree {
return node.tree
}
2015-05-02 14:12:19 +00:00
// NodeFromFileInfo returns a new node from the given path and FileInfo.
2015-02-03 20:07:55 +00:00
func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
Remove pools for nodes and IDs This removes the allocation pools for nodes and IDs. I feel they aren't really needed. Benchmarks: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 197890867 198616293 +0.37% BenchmarkChunkEncryptParallel 196127004 198819818 +1.37% BenchmarkArchiveDirectory 1098848419 1087237723 -1.06% BenchmarkPreload 30464455 29910239 -1.82% BenchmarkLoadTree 3265092 3088543 -5.41% BenchmarkEncryptWriter 37213511 37134683 -0.21% BenchmarkEncrypt 36037879 36166546 +0.36% BenchmarkDecryptReader 38165659 38556734 +1.02% BenchmarkEncryptDecryptReader 77027044 77194987 +0.22% BenchmarkDecrypt 36017602 35937888 -0.22% BenchmarkSaveJSON 47906 50270 +4.93% BenchmarkSaveFrom 49775973 50520969 +1.50% BenchmarkLoadJSONID 105290245 107281849 +1.89% BenchmarkChunkerWithSHA256 151501430 148264078 -2.14% BenchmarkChunkerWithMD5 93606346 94036392 +0.46% BenchmarkChunker 74285431 75933882 +2.22% BenchmarkPipelineWalker 387689 346467 -10.63% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 52.99 52.79 1.00x BenchmarkChunkEncryptParallel 53.46 52.74 0.99x BenchmarkEncryptWriter 225.42 225.90 1.00x BenchmarkEncrypt 232.77 231.94 1.00x BenchmarkDecryptReader 219.79 217.57 0.99x BenchmarkEncryptDecryptReader 108.90 108.67 1.00x BenchmarkDecrypt 232.90 233.42 1.00x BenchmarkSaveFrom 84.26 83.02 0.99x BenchmarkChunkerWithSHA256 69.21 70.72 1.02x BenchmarkChunkerWithMD5 112.02 111.51 1.00x BenchmarkChunker 141.15 138.09 0.98x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 110 110 +0.00% BenchmarkChunkEncryptParallel 100 100 +0.00% BenchmarkArchiveDirectory 475591 476635 +0.22% BenchmarkPreload 28059 24182 -13.82% BenchmarkLoadTree 3124 2889 -7.52% BenchmarkEncryptWriter 19 19 +0.00% BenchmarkEncrypt 13 13 +0.00% BenchmarkDecryptReader 16 15 -6.25% BenchmarkEncryptDecryptReader 39 39 +0.00% BenchmarkDecrypt 11 11 +0.00% BenchmarkSaveJSON 74 74 +0.00% BenchmarkSaveFrom 109 112 +2.75% BenchmarkLoadJSONID 103630 97849 -5.58% BenchmarkChunkerWithSHA256 13 13 +0.00% BenchmarkChunkerWithMD5 12 12 +0.00% BenchmarkChunker 6 6 +0.00% BenchmarkPipelineWalker 212 165 -22.17% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 64697 64697 +0.00% BenchmarkChunkEncryptParallel 64681 64681 +0.00% BenchmarkArchiveDirectory 193385504 193790864 +0.21% BenchmarkPreload 4064701 3942000 -3.02% BenchmarkLoadTree 344954 325396 -5.67% BenchmarkEncryptWriter 12793 12793 +0.00% BenchmarkEncrypt 1950 1950 +0.00% BenchmarkDecryptReader 3120 2774 -11.09% BenchmarkEncryptDecryptReader 1528036 1528036 +0.00% BenchmarkDecrypt 1919 1919 +0.00% BenchmarkSaveJSON 5524 5524 +0.00% BenchmarkSaveFrom 31353 40804 +30.14% BenchmarkLoadJSONID 12872020 16010968 +24.39% BenchmarkChunkerWithSHA256 26821 26821 +0.00% BenchmarkChunkerWithMD5 13554 13554 +0.00% BenchmarkChunker 13458 13458 +0.00% BenchmarkPipelineWalker 58584 55560 -5.16%
2015-03-22 15:38:03 +00:00
node := &Node{
path: path,
Name: fi.Name(),
Mode: fi.Mode() & (os.ModePerm | os.ModeType),
Remove pools for nodes and IDs This removes the allocation pools for nodes and IDs. I feel they aren't really needed. Benchmarks: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 197890867 198616293 +0.37% BenchmarkChunkEncryptParallel 196127004 198819818 +1.37% BenchmarkArchiveDirectory 1098848419 1087237723 -1.06% BenchmarkPreload 30464455 29910239 -1.82% BenchmarkLoadTree 3265092 3088543 -5.41% BenchmarkEncryptWriter 37213511 37134683 -0.21% BenchmarkEncrypt 36037879 36166546 +0.36% BenchmarkDecryptReader 38165659 38556734 +1.02% BenchmarkEncryptDecryptReader 77027044 77194987 +0.22% BenchmarkDecrypt 36017602 35937888 -0.22% BenchmarkSaveJSON 47906 50270 +4.93% BenchmarkSaveFrom 49775973 50520969 +1.50% BenchmarkLoadJSONID 105290245 107281849 +1.89% BenchmarkChunkerWithSHA256 151501430 148264078 -2.14% BenchmarkChunkerWithMD5 93606346 94036392 +0.46% BenchmarkChunker 74285431 75933882 +2.22% BenchmarkPipelineWalker 387689 346467 -10.63% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 52.99 52.79 1.00x BenchmarkChunkEncryptParallel 53.46 52.74 0.99x BenchmarkEncryptWriter 225.42 225.90 1.00x BenchmarkEncrypt 232.77 231.94 1.00x BenchmarkDecryptReader 219.79 217.57 0.99x BenchmarkEncryptDecryptReader 108.90 108.67 1.00x BenchmarkDecrypt 232.90 233.42 1.00x BenchmarkSaveFrom 84.26 83.02 0.99x BenchmarkChunkerWithSHA256 69.21 70.72 1.02x BenchmarkChunkerWithMD5 112.02 111.51 1.00x BenchmarkChunker 141.15 138.09 0.98x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 110 110 +0.00% BenchmarkChunkEncryptParallel 100 100 +0.00% BenchmarkArchiveDirectory 475591 476635 +0.22% BenchmarkPreload 28059 24182 -13.82% BenchmarkLoadTree 3124 2889 -7.52% BenchmarkEncryptWriter 19 19 +0.00% BenchmarkEncrypt 13 13 +0.00% BenchmarkDecryptReader 16 15 -6.25% BenchmarkEncryptDecryptReader 39 39 +0.00% BenchmarkDecrypt 11 11 +0.00% BenchmarkSaveJSON 74 74 +0.00% BenchmarkSaveFrom 109 112 +2.75% BenchmarkLoadJSONID 103630 97849 -5.58% BenchmarkChunkerWithSHA256 13 13 +0.00% BenchmarkChunkerWithMD5 12 12 +0.00% BenchmarkChunker 6 6 +0.00% BenchmarkPipelineWalker 212 165 -22.17% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 64697 64697 +0.00% BenchmarkChunkEncryptParallel 64681 64681 +0.00% BenchmarkArchiveDirectory 193385504 193790864 +0.21% BenchmarkPreload 4064701 3942000 -3.02% BenchmarkLoadTree 344954 325396 -5.67% BenchmarkEncryptWriter 12793 12793 +0.00% BenchmarkEncrypt 1950 1950 +0.00% BenchmarkDecryptReader 3120 2774 -11.09% BenchmarkEncryptDecryptReader 1528036 1528036 +0.00% BenchmarkDecrypt 1919 1919 +0.00% BenchmarkSaveJSON 5524 5524 +0.00% BenchmarkSaveFrom 31353 40804 +30.14% BenchmarkLoadJSONID 12872020 16010968 +24.39% BenchmarkChunkerWithSHA256 26821 26821 +0.00% BenchmarkChunkerWithMD5 13554 13554 +0.00% BenchmarkChunker 13458 13458 +0.00% BenchmarkPipelineWalker 58584 55560 -5.16%
2015-03-22 15:38:03 +00:00
ModTime: fi.ModTime(),
}
2015-02-03 20:07:55 +00:00
2015-04-29 01:24:43 +00:00
node.Type = nodeTypeFromFileInfo(fi)
2015-03-07 10:53:32 +00:00
if node.Type == "file" {
node.Size = uint64(fi.Size())
}
2015-04-24 23:39:32 +00:00
err := node.fillExtra(path, fi)
2015-03-07 10:53:32 +00:00
return node, err
}
2015-04-29 01:24:43 +00:00
func nodeTypeFromFileInfo(fi os.FileInfo) string {
2015-02-03 20:07:55 +00:00
switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
case 0:
2015-03-07 10:53:32 +00:00
return "file"
2015-02-03 20:07:55 +00:00
case os.ModeDir:
2015-03-07 10:53:32 +00:00
return "dir"
2015-02-03 20:07:55 +00:00
case os.ModeSymlink:
2015-03-07 10:53:32 +00:00
return "symlink"
2015-02-03 20:07:55 +00:00
case os.ModeDevice | os.ModeCharDevice:
2015-03-07 10:53:32 +00:00
return "chardev"
2015-02-03 20:07:55 +00:00
case os.ModeDevice:
2015-03-07 10:53:32 +00:00
return "dev"
2015-02-03 20:07:55 +00:00
case os.ModeNamedPipe:
2015-03-07 10:53:32 +00:00
return "fifo"
2015-02-03 20:07:55 +00:00
case os.ModeSocket:
2015-03-07 10:53:32 +00:00
return "socket"
2015-02-03 20:07:55 +00:00
}
2015-03-07 10:53:32 +00:00
return ""
2015-02-03 20:07:55 +00:00
}
2015-05-02 14:22:43 +00:00
// CreateAt creates the node at the given path and restores all the meta data.
2015-05-09 21:59:58 +00:00
func (node *Node) CreateAt(path string, repo *repository.Repository) error {
debug.Log("Node.CreateAt", "create node %v at %v", node.Name, path)
2015-02-03 20:07:55 +00:00
switch node.Type {
case "dir":
2015-04-29 01:24:43 +00:00
if err := node.createDirAt(path); err != nil {
return errors.Annotate(err, "createDirAt")
2015-02-03 20:07:55 +00:00
}
case "file":
2015-05-09 11:32:52 +00:00
if err := node.createFileAt(path, repo); err != nil {
return errors.Annotate(err, "createFileAt")
2015-02-03 20:07:55 +00:00
}
case "symlink":
2015-04-29 01:24:43 +00:00
if err := node.createSymlinkAt(path); err != nil {
return errors.Annotate(err, "createSymlinkAt")
2015-02-03 20:07:55 +00:00
}
case "dev":
2015-04-29 01:24:43 +00:00
if err := node.createDevAt(path); err != nil {
return errors.Annotate(err, "createDevAt")
2015-02-03 20:07:55 +00:00
}
case "chardev":
2015-04-29 01:24:43 +00:00
if err := node.createCharDevAt(path); err != nil {
return errors.Annotate(err, "createCharDevAt")
2015-02-03 20:07:55 +00:00
}
case "fifo":
2015-04-29 01:24:43 +00:00
if err := node.createFifoAt(path); err != nil {
return errors.Annotate(err, "createFifoAt")
2015-02-03 20:07:55 +00:00
}
case "socket":
return nil
default:
return fmt.Errorf("filetype %q not implemented!\n", node.Type)
}
err := node.restoreMetadata(path)
if err != nil {
debug.Log("Node.CreateAt", "restoreMetadata(%s) error %v", path, err)
}
return err
2015-04-29 01:24:43 +00:00
}
func (node Node) restoreMetadata(path string) error {
var err error
err = os.Lchown(path, int(node.UID), int(node.GID))
if err != nil {
return errors.Annotate(err, "Lchown")
2015-04-29 01:24:43 +00:00
}
if node.Type != "symlink" {
err = os.Chmod(path, node.Mode)
if err != nil {
return errors.Annotate(err, "Chmod")
}
2015-02-03 20:07:55 +00:00
}
2015-05-14 03:11:31 +00:00
if node.Type != "dir" {
err = node.RestoreTimestamps(path)
if err != nil {
debug.Log("Node.restoreMetadata", "error restoring timestamps for dir %v: %v", path, err)
2015-05-14 03:11:31 +00:00
return err
}
}
return nil
}
func (node Node) RestoreTimestamps(path string) error {
var utimes = [...]syscall.Timespec{
2015-04-29 01:24:43 +00:00
syscall.NsecToTimespec(node.AccessTime.UnixNano()),
syscall.NsecToTimespec(node.ModTime.UnixNano()),
}
2015-05-14 03:11:31 +00:00
if node.Type == "symlink" {
if err := node.restoreSymlinkTimestamps(path, utimes); err != nil {
return err
}
return nil
}
if err := syscall.UtimesNano(path, utimes[:]); err != nil {
return errors.Annotate(err, "UtimesNano")
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
}
func (node Node) createDirAt(path string) error {
err := os.Mkdir(path, node.Mode)
2015-02-03 20:07:55 +00:00
if err != nil {
return errors.Annotate(err, "Mkdir")
2015-02-03 20:07:55 +00:00
}
return nil
}
2015-05-09 21:59:58 +00:00
func (node Node) createFileAt(path string, repo *repository.Repository) error {
2015-04-29 01:24:43 +00:00
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()
if err != nil {
return errors.Annotate(err, "OpenFile")
2015-02-03 20:07:55 +00:00
}
for _, id := range node.Content {
2015-05-09 11:32:52 +00:00
buf, err := repo.LoadBlob(pack.Data, id)
2015-04-29 01:24:43 +00:00
if err != nil {
return errors.Annotate(err, "Load")
2015-04-29 01:24:43 +00:00
}
_, err = f.Write(buf)
if err != nil {
return errors.Annotate(err, "Write")
2015-04-29 01:24:43 +00:00
}
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
}
func (node Node) createSymlinkAt(path string) error {
err := os.Symlink(node.LinkTarget, path)
if err != nil {
return errors.Annotate(err, "Symlink")
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
return nil
2015-02-03 20:07:55 +00:00
}
func (node *Node) createDevAt(path string) error {
return syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device))
}
func (node *Node) createCharDevAt(path string) error {
return syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device))
}
func (node *Node) createFifoAt(path string) error {
return syscall.Mkfifo(path, 0600)
}
2015-02-03 20:07:55 +00:00
func (node Node) MarshalJSON() ([]byte, error) {
type nodeJSON Node
nj := nodeJSON(node)
name := strconv.Quote(node.Name)
nj.Name = name[1 : len(name)-1]
return json.Marshal(nj)
}
func (node *Node) UnmarshalJSON(data []byte) error {
type nodeJSON Node
2015-04-28 06:11:01 +00:00
nj := (*nodeJSON)(node)
2015-02-03 20:07:55 +00:00
err := json.Unmarshal(data, nj)
if err != nil {
return err
}
nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`)
return err
}
func (node Node) Equals(other Node) bool {
if node.Name != other.Name {
return false
}
if node.Type != other.Type {
return false
}
if node.Mode != other.Mode {
return false
}
if node.ModTime != other.ModTime {
return false
}
if node.AccessTime != other.AccessTime {
return false
}
if node.ChangeTime != other.ChangeTime {
return false
}
if node.UID != other.UID {
return false
}
if node.GID != other.GID {
return false
}
if node.User != other.User {
return false
}
if node.Group != other.Group {
return false
}
if node.Inode != other.Inode {
return false
}
if node.Size != other.Size {
return false
}
if node.Links != other.Links {
return false
}
if node.LinkTarget != other.LinkTarget {
return false
}
if node.Device != other.Device {
return false
}
2015-04-29 01:24:43 +00:00
if !node.sameContent(other) {
2015-02-03 20:07:55 +00:00
return false
2015-04-29 01:24:43 +00:00
}
if !node.Subtree.Equal(other.Subtree) {
2015-02-03 20:07:55 +00:00
return false
2015-04-29 01:24:43 +00:00
}
if node.Error != other.Error {
return false
}
2015-02-03 20:07:55 +00:00
2015-04-29 01:24:43 +00:00
return true
}
func (node Node) sameContent(other Node) bool {
if node.Content == nil {
return other.Content == nil
2015-02-03 20:07:55 +00:00
}
2015-04-29 01:24:43 +00:00
if other.Content == nil {
2015-02-03 20:07:55 +00:00
return false
}
2015-04-29 01:24:43 +00:00
if len(node.Content) != len(other.Content) {
2015-02-03 20:07:55 +00:00
return false
}
2015-04-29 01:24:43 +00:00
for i := 0; i < len(node.Content); i++ {
if !node.Content[i].Equal(other.Content[i]) {
return false
}
}
2015-02-03 20:07:55 +00:00
return true
}
2015-04-29 03:31:07 +00:00
func (node *Node) isNewer(path string, fi os.FileInfo) bool {
if node.Type != "file" {
debug.Log("node.isNewer", "node %v is newer: not file", path)
return true
}
tpe := nodeTypeFromFileInfo(fi)
if node.Name != fi.Name() || node.Type != tpe {
debug.Log("node.isNewer", "node %v is newer: name or type changed", path)
return true
}
extendedStat := fi.Sys().(*syscall.Stat_t)
inode := extendedStat.Ino
size := uint64(extendedStat.Size)
if node.ModTime != fi.ModTime() ||
node.ChangeTime != changeTime(extendedStat) ||
node.Inode != uint64(inode) ||
2015-04-29 03:31:07 +00:00
node.Size != size {
debug.Log("node.isNewer", "node %v is newer: timestamp, size or inode changed", path)
return true
}
debug.Log("node.isNewer", "node %v is not newer", path)
return false
}
2015-04-29 03:45:24 +00:00
func (node *Node) fillUser(stat *syscall.Stat_t) error {
node.UID = stat.Uid
node.GID = stat.Gid
2015-05-01 21:26:40 +00:00
username, err := lookupUsername(strconv.Itoa(int(stat.Uid)))
2015-04-29 03:45:24 +00:00
if err != nil {
return errors.Annotate(err, "fillUser")
2015-04-29 03:45:24 +00:00
}
2015-05-01 21:26:40 +00:00
node.User = username
2015-04-29 03:45:24 +00:00
return nil
}
2015-05-01 21:26:40 +00:00
var (
uidLookupCache = make(map[string]string)
uidLookupCacheMutex = sync.RWMutex{}
)
func lookupUsername(uid string) (string, error) {
uidLookupCacheMutex.RLock()
value, ok := uidLookupCache[uid]
uidLookupCacheMutex.RUnlock()
if ok {
return value, nil
}
username := ""
2015-05-01 21:26:40 +00:00
u, err := user.LookupId(uid)
if err == nil {
username = u.Username
2015-05-01 21:26:40 +00:00
}
uidLookupCacheMutex.Lock()
uidLookupCache[uid] = username
2015-05-01 21:26:40 +00:00
uidLookupCacheMutex.Unlock()
return username, nil
2015-05-01 21:26:40 +00:00
}
2015-04-29 03:45:24 +00:00
func (node *Node) fillExtra(path string, fi os.FileInfo) error {
stat, ok := fi.Sys().(*syscall.Stat_t)
if !ok {
return nil
}
node.Inode = uint64(stat.Ino)
2015-04-29 03:45:24 +00:00
node.fillTimes(stat)
var err error
2015-05-01 21:31:02 +00:00
if err = node.fillUser(stat); err != nil {
return errors.Annotate(err, "fillExtra")
2015-05-01 21:31:02 +00:00
}
2015-04-29 03:45:24 +00:00
switch node.Type {
case "file":
node.Size = uint64(stat.Size)
node.Links = uint64(stat.Nlink)
case "dir":
case "symlink":
node.LinkTarget, err = os.Readlink(path)
case "dev":
node.Device = uint64(stat.Rdev)
2015-04-29 03:45:24 +00:00
case "chardev":
node.Device = uint64(stat.Rdev)
2015-04-29 03:45:24 +00:00
case "fifo":
case "socket":
default:
err = fmt.Errorf("invalid node type %q", node.Type)
}
return err
}