From 460ebebeef1f9c410e50f23a306d84b168ef3922 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 3 Feb 2015 21:07:55 +0100 Subject: [PATCH] Split out Node and Blob from tree.go --- blob.go | 41 ++++++ node.go | 379 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tree.go | 406 -------------------------------------------------------- 3 files changed, 420 insertions(+), 406 deletions(-) create mode 100644 blob.go create mode 100644 node.go diff --git a/blob.go b/blob.go new file mode 100644 index 000000000..64bb94567 --- /dev/null +++ b/blob.go @@ -0,0 +1,41 @@ +package restic + +import ( + "fmt" + + "github.com/restic/restic/backend" +) + +type Blob struct { + ID backend.ID `json:"id,omitempty"` + Offset uint64 `json:"offset,omitempty"` + Size uint64 `json:"size,omitempty"` + Storage backend.ID `json:"sid,omitempty"` // encrypted ID + StorageSize uint64 `json:"ssize,omitempty"` // encrypted Size +} + +type Blobs []Blob + +func (b Blob) Free() { + if b.ID != nil { + b.ID.Free() + } + + if b.Storage != nil { + b.Storage.Free() + } +} + +func (b Blob) Valid() bool { + if b.ID == nil || b.Storage == nil || b.StorageSize == 0 { + return false + } + + return true +} + +func (b Blob) String() string { + return fmt.Sprintf("Blob<%s -> %s>", + b.ID.Str(), + b.Storage.Str()) +} diff --git a/node.go b/node.go new file mode 100644 index 000000000..10907ea3c --- /dev/null +++ b/node.go @@ -0,0 +1,379 @@ +package restic + +import ( + "encoding/json" + "fmt" + "os" + "os/user" + "strconv" + "syscall" + "time" + + "github.com/juju/arrar" + "github.com/restic/restic/backend" +) + +type Node struct { + Name string `json:"name"` + Type string `json:"type"` + Mode os.FileMode `json:"mode,omitempty"` + ModTime time.Time `json:"mtime,omitempty"` + AccessTime time.Time `json:"atime,omitempty"` + ChangeTime time.Time `json:"ctime,omitempty"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Size uint64 `json:"size,omitempty"` + Links uint64 `json:"links,omitempty"` + LinkTarget string `json:"linktarget,omitempty"` + Device uint64 `json:"device,omitempty"` + Content []backend.ID `json:"content"` + Subtree backend.ID `json:"subtree,omitempty"` + + Error string `json:"error,omitempty"` + + tree *Tree + + path string + err error +} + +func (n Node) String() string { + switch n.Type { + case "file": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name) + case "dir": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name) + } + + return fmt.Sprintf("", n.Type, n.Name) +} + +func (node Node) Tree() *Tree { + return node.tree +} + +func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { + stat, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + return + } + + node.ChangeTime = time.Unix(stat.Ctim.Unix()) + node.AccessTime = time.Unix(stat.Atim.Unix()) + node.UID = stat.Uid + node.GID = stat.Gid + + // TODO: cache uid lookup + if u, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { + node.User = u.Username + } + + // TODO: implement getgrnam() or use https://github.com/kless/osutil + // if g, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { + // node.User = u.Username + // } + + node.Inode = stat.Ino + + switch node.Type { + case "file": + node.Size = uint64(stat.Size) + node.Links = stat.Nlink + case "dir": + // nothing to do + case "symlink": + node.LinkTarget, err = os.Readlink(path) + case "dev": + node.Device = stat.Rdev + case "chardev": + node.Device = stat.Rdev + case "fifo": + // nothing to do + case "socket": + // nothing to do + default: + panic(fmt.Sprintf("invalid node type %q", node.Type)) + } + + return err +} + +func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { + node := GetNode() + node.path = path + node.Name = fi.Name() + node.Mode = fi.Mode() & os.ModePerm + node.ModTime = fi.ModTime() + + switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { + case 0: + node.Type = "file" + case os.ModeDir: + node.Type = "dir" + case os.ModeSymlink: + node.Type = "symlink" + case os.ModeDevice | os.ModeCharDevice: + node.Type = "chardev" + case os.ModeDevice: + node.Type = "dev" + case os.ModeNamedPipe: + node.Type = "fifo" + case os.ModeSocket: + node.Type = "socket" + } + + err := node.fill_extra(path, fi) + return node, err +} + +func CreateNodeAt(node *Node, m *Map, s Server, path string) error { + switch node.Type { + case "dir": + err := os.Mkdir(path, node.Mode) + if err != nil { + return arrar.Annotate(err, "Mkdir") + } + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return arrar.Annotate(err, "Lchown") + } + + var utimes = []syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + err = syscall.UtimesNano(path, utimes) + if err != nil { + return arrar.Annotate(err, "Utimesnano") + } + case "file": + // TODO: handle hard links + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) + defer f.Close() + if err != nil { + return arrar.Annotate(err, "OpenFile") + } + + for _, blobid := range node.Content { + blob, err := m.FindID(blobid) + if err != nil { + return arrar.Annotate(err, "Find Blob") + } + + buf, err := s.Load(backend.Data, blob) + if err != nil { + return arrar.Annotate(err, "Load") + } + + _, err = f.Write(buf) + if err != nil { + return arrar.Annotate(err, "Write") + } + } + + f.Close() + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return arrar.Annotate(err, "Lchown") + } + + var utimes = []syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + err = syscall.UtimesNano(path, utimes) + if err != nil { + return arrar.Annotate(err, "Utimesnano") + } + case "symlink": + err := os.Symlink(node.LinkTarget, path) + if err != nil { + return arrar.Annotate(err, "Symlink") + } + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return arrar.Annotate(err, "Lchown") + } + + f, err := os.OpenFile(path, O_PATH|syscall.O_NOFOLLOW, 0600) + defer f.Close() + if err != nil { + return arrar.Annotate(err, "OpenFile") + } + + // TODO: Get Futimes() working on older Linux kernels (fails with 3.2.0) + // var utimes = []syscall.Timeval{ + // syscall.NsecToTimeval(node.AccessTime.UnixNano()), + // syscall.NsecToTimeval(node.ModTime.UnixNano()), + // } + // err = syscall.Futimes(int(f.Fd()), utimes) + // if err != nil { + // return arrar.Annotate(err, "Futimes") + // } + + return nil + case "dev": + err := syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device)) + if err != nil { + return arrar.Annotate(err, "Mknod") + } + case "chardev": + err := syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device)) + if err != nil { + return arrar.Annotate(err, "Mknod") + } + case "fifo": + err := syscall.Mkfifo(path, 0600) + if err != nil { + return arrar.Annotate(err, "Mkfifo") + } + case "socket": + // nothing to do, we do not restore sockets + return nil + default: + return fmt.Errorf("filetype %q not implemented!\n", node.Type) + } + + err := os.Chmod(path, node.Mode) + if err != nil { + return arrar.Annotate(err, "Chmod") + } + + err = os.Chown(path, int(node.UID), int(node.GID)) + if err != nil { + return arrar.Annotate(err, "Chown") + } + + err = os.Chtimes(path, node.AccessTime, node.ModTime) + if err != nil { + return arrar.Annotate(err, "Chtimes") + } + + return nil +} + +func (node Node) SameContent(olderNode *Node) bool { + // if this node has a type other than "file", treat as if content has changed + if node.Type != "file" { + return false + } + + // if the name or type has changed, this is surely something different + if node.Name != olderNode.Name || node.Type != olderNode.Type { + return false + } + + // if timestamps or inodes differ, content has changed + if node.ModTime != olderNode.ModTime || + node.ChangeTime != olderNode.ChangeTime || + node.Inode != olderNode.Inode { + return false + } + + // otherwise the node is assumed to have the same content + return true +} + +func (node Node) MarshalJSON() ([]byte, error) { + type nodeJSON Node + nj := nodeJSON(node) + name := strconv.Quote(node.Name) + nj.Name = name[1 : len(name)-1] + + return json.Marshal(nj) +} + +func (node *Node) UnmarshalJSON(data []byte) error { + type nodeJSON Node + var nj *nodeJSON = (*nodeJSON)(node) + + err := json.Unmarshal(data, nj) + if err != nil { + return err + } + + nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`) + return err +} + +func (node Node) Equals(other Node) bool { + // TODO: add generatored code for this + if node.Name != other.Name { + return false + } + if node.Type != other.Type { + return false + } + if node.Mode != other.Mode { + return false + } + if node.ModTime != other.ModTime { + return false + } + if node.AccessTime != other.AccessTime { + return false + } + if node.ChangeTime != other.ChangeTime { + return false + } + if node.UID != other.UID { + return false + } + if node.GID != other.GID { + return false + } + if node.User != other.User { + return false + } + if node.Group != other.Group { + return false + } + if node.Inode != other.Inode { + return false + } + if node.Size != other.Size { + return false + } + if node.Links != other.Links { + return false + } + if node.LinkTarget != other.LinkTarget { + return false + } + if node.Device != other.Device { + return false + } + if node.Content != nil && other.Content == nil { + return false + } else if node.Content == nil && other.Content != nil { + return false + } else if node.Content != nil && other.Content != nil { + if len(node.Content) != len(other.Content) { + return false + } + + for i := 0; i < len(node.Content); i++ { + if !node.Content[i].Equal(other.Content[i]) { + return false + } + } + } + + if !node.Subtree.Equal(other.Subtree) { + return false + } + + if node.Error != other.Error { + return false + } + + return true +} diff --git a/tree.go b/tree.go index 858aa1f22..b916f7339 100644 --- a/tree.go +++ b/tree.go @@ -1,18 +1,11 @@ package restic import ( - "encoding/json" "errors" "fmt" - "os" - "os/user" "path/filepath" "sort" - "strconv" - "syscall" - "time" - "github.com/juju/arrar" "github.com/restic/restic/backend" "github.com/restic/restic/debug" ) @@ -22,61 +15,11 @@ type Tree struct { Map *Map `json:"map"` } -type Node struct { - Name string `json:"name"` - Type string `json:"type"` - Mode os.FileMode `json:"mode,omitempty"` - ModTime time.Time `json:"mtime,omitempty"` - AccessTime time.Time `json:"atime,omitempty"` - ChangeTime time.Time `json:"ctime,omitempty"` - UID uint32 `json:"uid"` - GID uint32 `json:"gid"` - User string `json:"user,omitempty"` - Group string `json:"group,omitempty"` - Inode uint64 `json:"inode,omitempty"` - Size uint64 `json:"size,omitempty"` - Links uint64 `json:"links,omitempty"` - LinkTarget string `json:"linktarget,omitempty"` - Device uint64 `json:"device,omitempty"` - Content []backend.ID `json:"content"` - Subtree backend.ID `json:"subtree,omitempty"` - - Error string `json:"error,omitempty"` - - tree *Tree - - path string - err error -} - var ( ErrNodeNotFound = errors.New("named node not found") ErrNodeAlreadyInTree = errors.New("node already present") ) -type Blob struct { - ID backend.ID `json:"id,omitempty"` - Offset uint64 `json:"offset,omitempty"` - Size uint64 `json:"size,omitempty"` - Storage backend.ID `json:"sid,omitempty"` // encrypted ID - StorageSize uint64 `json:"ssize,omitempty"` // encrypted Size -} - -type Blobs []Blob - -func (n Node) String() string { - switch n.Type { - case "file": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name) - case "dir": - return fmt.Sprintf("%s %5d %5d %6d %s %s", - n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name) - } - - return fmt.Sprintf("", n.Type, n.Name) -} - func NewTree() *Tree { return &Tree{ Nodes: []*Node{}, @@ -304,352 +247,3 @@ func (t Tree) StatTodo() Stat { return s } - -func (node Node) Tree() *Tree { - return node.tree -} - -func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { - stat, ok := fi.Sys().(*syscall.Stat_t) - if !ok { - return - } - - node.ChangeTime = time.Unix(stat.Ctim.Unix()) - node.AccessTime = time.Unix(stat.Atim.Unix()) - node.UID = stat.Uid - node.GID = stat.Gid - - // TODO: cache uid lookup - if u, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { - node.User = u.Username - } - - // TODO: implement getgrnam() or use https://github.com/kless/osutil - // if g, nil := user.LookupId(strconv.Itoa(int(stat.Uid))); err == nil { - // node.User = u.Username - // } - - node.Inode = stat.Ino - - switch node.Type { - case "file": - node.Size = uint64(stat.Size) - node.Links = stat.Nlink - case "dir": - // nothing to do - case "symlink": - node.LinkTarget, err = os.Readlink(path) - case "dev": - node.Device = stat.Rdev - case "chardev": - node.Device = stat.Rdev - case "fifo": - // nothing to do - case "socket": - // nothing to do - default: - panic(fmt.Sprintf("invalid node type %q", node.Type)) - } - - return err -} - -func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { - node := GetNode() - node.path = path - node.Name = fi.Name() - node.Mode = fi.Mode() & os.ModePerm - node.ModTime = fi.ModTime() - - switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { - case 0: - node.Type = "file" - case os.ModeDir: - node.Type = "dir" - case os.ModeSymlink: - node.Type = "symlink" - case os.ModeDevice | os.ModeCharDevice: - node.Type = "chardev" - case os.ModeDevice: - node.Type = "dev" - case os.ModeNamedPipe: - node.Type = "fifo" - case os.ModeSocket: - node.Type = "socket" - } - - err := node.fill_extra(path, fi) - return node, err -} - -func CreateNodeAt(node *Node, m *Map, s Server, path string) error { - switch node.Type { - case "dir": - err := os.Mkdir(path, node.Mode) - if err != nil { - return arrar.Annotate(err, "Mkdir") - } - - err = os.Lchown(path, int(node.UID), int(node.GID)) - if err != nil { - return arrar.Annotate(err, "Lchown") - } - - var utimes = []syscall.Timespec{ - syscall.NsecToTimespec(node.AccessTime.UnixNano()), - syscall.NsecToTimespec(node.ModTime.UnixNano()), - } - err = syscall.UtimesNano(path, utimes) - if err != nil { - return arrar.Annotate(err, "Utimesnano") - } - case "file": - // TODO: handle hard links - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) - defer f.Close() - if err != nil { - return arrar.Annotate(err, "OpenFile") - } - - for _, blobid := range node.Content { - blob, err := m.FindID(blobid) - if err != nil { - return arrar.Annotate(err, "Find Blob") - } - - buf, err := s.Load(backend.Data, blob) - if err != nil { - return arrar.Annotate(err, "Load") - } - - _, err = f.Write(buf) - if err != nil { - return arrar.Annotate(err, "Write") - } - } - - f.Close() - - err = os.Lchown(path, int(node.UID), int(node.GID)) - if err != nil { - return arrar.Annotate(err, "Lchown") - } - - var utimes = []syscall.Timespec{ - syscall.NsecToTimespec(node.AccessTime.UnixNano()), - syscall.NsecToTimespec(node.ModTime.UnixNano()), - } - err = syscall.UtimesNano(path, utimes) - if err != nil { - return arrar.Annotate(err, "Utimesnano") - } - case "symlink": - err := os.Symlink(node.LinkTarget, path) - if err != nil { - return arrar.Annotate(err, "Symlink") - } - - err = os.Lchown(path, int(node.UID), int(node.GID)) - if err != nil { - return arrar.Annotate(err, "Lchown") - } - - f, err := os.OpenFile(path, O_PATH|syscall.O_NOFOLLOW, 0600) - defer f.Close() - if err != nil { - return arrar.Annotate(err, "OpenFile") - } - - // TODO: Get Futimes() working on older Linux kernels (fails with 3.2.0) - // var utimes = []syscall.Timeval{ - // syscall.NsecToTimeval(node.AccessTime.UnixNano()), - // syscall.NsecToTimeval(node.ModTime.UnixNano()), - // } - // err = syscall.Futimes(int(f.Fd()), utimes) - // if err != nil { - // return arrar.Annotate(err, "Futimes") - // } - - return nil - case "dev": - err := syscall.Mknod(path, syscall.S_IFBLK|0600, int(node.Device)) - if err != nil { - return arrar.Annotate(err, "Mknod") - } - case "chardev": - err := syscall.Mknod(path, syscall.S_IFCHR|0600, int(node.Device)) - if err != nil { - return arrar.Annotate(err, "Mknod") - } - case "fifo": - err := syscall.Mkfifo(path, 0600) - if err != nil { - return arrar.Annotate(err, "Mkfifo") - } - case "socket": - // nothing to do, we do not restore sockets - return nil - default: - return fmt.Errorf("filetype %q not implemented!\n", node.Type) - } - - err := os.Chmod(path, node.Mode) - if err != nil { - return arrar.Annotate(err, "Chmod") - } - - err = os.Chown(path, int(node.UID), int(node.GID)) - if err != nil { - return arrar.Annotate(err, "Chown") - } - - err = os.Chtimes(path, node.AccessTime, node.ModTime) - if err != nil { - return arrar.Annotate(err, "Chtimes") - } - - return nil -} - -func (node Node) SameContent(olderNode *Node) bool { - // if this node has a type other than "file", treat as if content has changed - if node.Type != "file" { - return false - } - - // if the name or type has changed, this is surely something different - if node.Name != olderNode.Name || node.Type != olderNode.Type { - return false - } - - // if timestamps or inodes differ, content has changed - if node.ModTime != olderNode.ModTime || - node.ChangeTime != olderNode.ChangeTime || - node.Inode != olderNode.Inode { - return false - } - - // otherwise the node is assumed to have the same content - return true -} - -func (node Node) MarshalJSON() ([]byte, error) { - type nodeJSON Node - nj := nodeJSON(node) - name := strconv.Quote(node.Name) - nj.Name = name[1 : len(name)-1] - - return json.Marshal(nj) -} - -func (node *Node) UnmarshalJSON(data []byte) error { - type nodeJSON Node - var nj *nodeJSON = (*nodeJSON)(node) - - err := json.Unmarshal(data, nj) - if err != nil { - return err - } - - nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`) - return err -} - -func (node Node) Equals(other Node) bool { - // TODO: add generatored code for this - if node.Name != other.Name { - return false - } - if node.Type != other.Type { - return false - } - if node.Mode != other.Mode { - return false - } - if node.ModTime != other.ModTime { - return false - } - if node.AccessTime != other.AccessTime { - return false - } - if node.ChangeTime != other.ChangeTime { - return false - } - if node.UID != other.UID { - return false - } - if node.GID != other.GID { - return false - } - if node.User != other.User { - return false - } - if node.Group != other.Group { - return false - } - if node.Inode != other.Inode { - return false - } - if node.Size != other.Size { - return false - } - if node.Links != other.Links { - return false - } - if node.LinkTarget != other.LinkTarget { - return false - } - if node.Device != other.Device { - return false - } - if node.Content != nil && other.Content == nil { - return false - } else if node.Content == nil && other.Content != nil { - return false - } else if node.Content != nil && other.Content != nil { - if len(node.Content) != len(other.Content) { - return false - } - - for i := 0; i < len(node.Content); i++ { - if !node.Content[i].Equal(other.Content[i]) { - return false - } - } - } - - if !node.Subtree.Equal(other.Subtree) { - return false - } - - if node.Error != other.Error { - return false - } - - return true -} - -func (b Blob) Free() { - if b.ID != nil { - b.ID.Free() - } - - if b.Storage != nil { - b.Storage.Free() - } -} - -func (b Blob) Valid() bool { - if b.ID == nil || b.Storage == nil || b.StorageSize == 0 { - return false - } - - return true -} - -func (b Blob) String() string { - return fmt.Sprintf("Blob<%s -> %s>", - b.ID.Str(), - b.Storage.Str()) -}