From 02200acad0a1194cc0ebf6cb4292627ceb8359f5 Mon Sep 17 00:00:00 2001 From: Tobias Klein Date: Thu, 5 Oct 2017 14:55:55 +0200 Subject: [PATCH] fuse: mount and backup in parallel (#1330) --- cmd/restic/cmd_mount.go | 6 + internal/fuse/root.go | 52 +--- internal/fuse/snapshots_dir.go | 421 ++++++++++++++++++++++++++++----- 3 files changed, 378 insertions(+), 101 deletions(-) diff --git a/cmd/restic/cmd_mount.go b/cmd/restic/cmd_mount.go index 8f24fdcc9..b38d13cdb 100644 --- a/cmd/restic/cmd_mount.go +++ b/cmd/restic/cmd_mount.go @@ -67,6 +67,12 @@ func mount(opts MountOptions, gopts GlobalOptions, mountpoint string) error { return err } + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + err = repo.LoadIndex(context.TODO()) if err != nil { return err diff --git a/internal/fuse/root.go b/internal/fuse/root.go index d59e36f58..8ce9e6605 100644 --- a/internal/fuse/root.go +++ b/internal/fuse/root.go @@ -41,22 +41,18 @@ const rootInode = 1 func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, error) { debug.Log("NewRoot(), config %v", cfg) - snapshots := restic.FindFilteredSnapshots(ctx, repo, cfg.Host, cfg.Tags, cfg.Paths) - debug.Log("found %d matching snapshots", len(snapshots)) - root := &Root{ repo: repo, inode: rootInode, cfg: cfg, - snapshots: snapshots, blobSizeCache: NewBlobSizeCache(ctx, repo.Index()), } entries := map[string]fs.Node{ - "snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), snapshots), - "tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags"), snapshots), - "hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts"), snapshots), - "ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids"), snapshots), + "snapshots": NewSnapshotsDir(root, fs.GenerateDynamicInode(root.inode, "snapshots"), "", ""), + "tags": NewTagsDir(root, fs.GenerateDynamicInode(root.inode, "tags")), + "hosts": NewHostsDir(root, fs.GenerateDynamicInode(root.inode, "hosts")), + "ids": NewSnapshotsIDSDir(root, fs.GenerateDynamicInode(root.inode, "ids")), } root.MetaDir = NewMetaDir(root, rootInode, entries) @@ -64,46 +60,6 @@ func NewRoot(ctx context.Context, repo restic.Repository, cfg Config) (*Root, er return root, nil } -// NewTagsDir returns a new directory containing entries, which in turn contains -// snapshots with this tag set. -func NewTagsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node { - tags := make(map[string]restic.Snapshots) - for _, sn := range snapshots { - for _, tag := range sn.Tags { - tags[tag] = append(tags[tag], sn) - } - } - - debug.Log("create tags dir with %d tags, inode %d", len(tags), inode) - - entries := make(map[string]fs.Node) - for name, snapshots := range tags { - debug.Log(" tag %v has %v snapshots", name, len(snapshots)) - entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots) - } - - return NewMetaDir(root, inode, entries) -} - -// NewHostsDir returns a new directory containing hostnames, which in -// turn contains snapshots of a single host each. -func NewHostsDir(root *Root, inode uint64, snapshots restic.Snapshots) fs.Node { - hosts := make(map[string]restic.Snapshots) - for _, sn := range snapshots { - hosts[sn.Hostname] = append(hosts[sn.Hostname], sn) - } - - debug.Log("create hosts dir with %d snapshots, inode %d", len(hosts), inode) - - entries := make(map[string]fs.Node) - for name, snapshots := range hosts { - debug.Log(" host %v has %v snapshots", name, len(snapshots)) - entries[name] = NewSnapshotsDir(root, fs.GenerateDynamicInode(inode, name), snapshots) - } - - return NewMetaDir(root, inode, entries) -} - // Root is just there to satisfy fs.Root, it returns itself. func (r *Root) Root() (fs.Node, error) { debug.Log("Root()") diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 467b0ceae..f468cb2cc 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -17,70 +17,128 @@ import ( "bazil.org/fuse/fs" ) -// SnapshotsDir is a fuse directory which contains snapshots. +// SnapshotsDir is a fuse directory which contains snapshots named by timestamp. type SnapshotsDir struct { - inode uint64 - root *Root - snapshots restic.Snapshots - names map[string]*restic.Snapshot - latest string + inode uint64 + root *Root + names map[string]*restic.Snapshot + latest string + tag string + host string +} + +// SnapshotsIDSDir is a fuse directory which contains snapshots named by ids. +type SnapshotsIDSDir struct { + inode uint64 + root *Root + names map[string]*restic.Snapshot +} + +// HostsDir is a fuse directory which contains hosts. +type HostsDir struct { + inode uint64 + root *Root + hosts map[string]bool +} + +// TagsDir is a fuse directory which contains tags. +type TagsDir struct { + inode uint64 + root *Root + tags map[string]bool +} + +// SnapshotLink +type snapshotLink struct { + root *Root + inode uint64 + target string + snapshot *restic.Snapshot } // ensure that *SnapshotsDir implements these interfaces var _ = fs.HandleReadDirAller(&SnapshotsDir{}) var _ = fs.NodeStringLookuper(&SnapshotsDir{}) +var _ = fs.HandleReadDirAller(&SnapshotsIDSDir{}) +var _ = fs.NodeStringLookuper(&SnapshotsIDSDir{}) +var _ = fs.HandleReadDirAller(&TagsDir{}) +var _ = fs.NodeStringLookuper(&TagsDir{}) +var _ = fs.HandleReadDirAller(&HostsDir{}) +var _ = fs.NodeStringLookuper(&HostsDir{}) var _ = fs.NodeReadlinker(&snapshotLink{}) -// NewSnapshotsDir returns a new directory containing snapshots. -func NewSnapshotsDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir { - debug.Log("create snapshots dir with %d snapshots, inode %d", len(snapshots), inode) - d := &SnapshotsDir{ - root: root, - inode: inode, - snapshots: snapshots, - names: make(map[string]*restic.Snapshot, len(snapshots)), +// read tag names from the current repository-state. +func getTagNames(d *TagsDir) { + d.tags = make(map[string]bool, len(d.root.snapshots)) + for _, snapshot := range d.root.snapshots { + for _, tag := range snapshot.Tags { + d.tags[tag] = true + } } +} - // Track latest Snapshot - var latestTime time.Time - d.latest = "" - - for _, sn := range snapshots { - name := sn.Time.Format(time.RFC3339) - if d.latest == "" || !sn.Time.Before(latestTime) { - latestTime = sn.Time - d.latest = name - } - for i := 1; ; i++ { - if _, ok := d.names[name]; !ok { - break - } - - name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i) - } +// read host names from the current repository-state. +func getHostsNames(d *HostsDir) { + d.hosts = make(map[string]bool, len(d.root.snapshots)) + for _, snapshot := range d.root.snapshots { + d.hosts[snapshot.Hostname] = true + } +} +// read snapshot id names from the current repository-state. +func getSnapshotIDSNames(d *SnapshotsIDSDir) { + for _, sn := range d.root.snapshots { + name := sn.ID().Str() d.names[name] = sn - debug.Log(" add snapshot %v as dir %v", sn.ID().Str(), name) + } +} + +// NewSnapshotsDir returns a new directory containing snapshots. +func NewSnapshotsDir(root *Root, inode uint64, tag string, host string) *SnapshotsDir { + debug.Log("create snapshots dir, inode %d", inode) + d := &SnapshotsDir{ + root: root, + inode: inode, + names: make(map[string]*restic.Snapshot), + latest: "", + tag: tag, + host: host, } return d } // NewSnapshotsIDSDir returns a new directory containing snapshots named by ids. -func NewSnapshotsIDSDir(root *Root, inode uint64, snapshots restic.Snapshots) *SnapshotsDir { - debug.Log("create snapshots ids dir with %d snapshots, inode %d", len(snapshots), inode) - d := &SnapshotsDir{ - root: root, - inode: inode, - snapshots: snapshots, - names: make(map[string]*restic.Snapshot, len(snapshots)), +func NewSnapshotsIDSDir(root *Root, inode uint64) *SnapshotsIDSDir { + debug.Log("create snapshots ids dir, inode %d", inode) + d := &SnapshotsIDSDir{ + root: root, + inode: inode, + names: make(map[string]*restic.Snapshot), } - for _, sn := range snapshots { - name := sn.ID().Str() + return d +} - d.names[name] = sn - debug.Log(" add snapshot %v", name) +// NewHostsDir returns a new directory containing host names +func NewHostsDir(root *Root, inode uint64) *HostsDir { + debug.Log("create hosts dir, inode %d", inode) + d := &HostsDir{ + root: root, + inode: inode, + hosts: make(map[string]bool), + } + + return d +} + +// NewTagsDir returns a new directory containing tag names +func NewTagsDir(root *Root, inode uint64) *TagsDir { + debug.Log("create tags dir, inode %d", inode) + d := &TagsDir{ + root: root, + inode: inode, + tags: make(map[string]bool), } return d @@ -99,9 +157,91 @@ func (d *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error { return nil } -// ReadDirAll returns all entries of the root node. +// Attr returns the attributes for the SnapshotsDir. +func (d *SnapshotsIDSDir) Attr(ctx context.Context, attr *fuse.Attr) error { + attr.Inode = d.inode + attr.Mode = os.ModeDir | 0555 + + if !d.root.cfg.OwnerIsRoot { + attr.Uid = uint32(os.Getuid()) + attr.Gid = uint32(os.Getgid()) + } + debug.Log("attr: %v", attr) + return nil +} + +// Attr returns the attributes for the HostsDir. +func (d *HostsDir) Attr(ctx context.Context, attr *fuse.Attr) error { + attr.Inode = d.inode + attr.Mode = os.ModeDir | 0555 + + if !d.root.cfg.OwnerIsRoot { + attr.Uid = uint32(os.Getuid()) + attr.Gid = uint32(os.Getgid()) + } + debug.Log("attr: %v", attr) + return nil +} + +// Attr returns the attributes for the TagsDir. +func (d *TagsDir) Attr(ctx context.Context, attr *fuse.Attr) error { + attr.Inode = d.inode + attr.Mode = os.ModeDir | 0555 + + if !d.root.cfg.OwnerIsRoot { + attr.Uid = uint32(os.Getuid()) + attr.Gid = uint32(os.Getgid()) + } + debug.Log("attr: %v", attr) + return nil +} + +// search element in string list. +func isElem(e string, list []string) bool { + for _, x := range list { + if e == x { + return true + } + } + return false +} + +// read snapshot timestamps from the current repository-state. +func getSnapshotNames(d *SnapshotsDir) { + var latestTime time.Time + d.latest = "" + d.names = make(map[string]*restic.Snapshot, len(d.root.snapshots)) + for _, sn := range d.root.snapshots { + if d.tag == "" || isElem(d.tag, sn.Tags) { + if d.host == "" || d.host == sn.Hostname { + name := sn.Time.Format(time.RFC3339) + if d.latest == "" || !sn.Time.Before(latestTime) { + latestTime = sn.Time + d.latest = name + } + for i := 1; ; i++ { + if _, ok := d.names[name]; !ok { + break + } + + name = fmt.Sprintf("%s-%d", sn.Time.Format(time.RFC3339), i) + } + + d.names[name] = sn + } + } + } +} + +// ReadDirAll returns all entries of the SnapshotsDir. func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { debug.Log("ReadDirAll()") + + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getSnapshotNames(d) + items := []fuse.Dirent{ { Inode: d.inode, @@ -134,21 +274,116 @@ func (d *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { return items, nil } -type snapshotLink struct { - root *Root - inode uint64 - target string - snapshot *restic.Snapshot +// ReadDirAll returns all entries of the SnapshotsIDSDir. +func (d *SnapshotsIDSDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + debug.Log("ReadDirAll()") + + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getSnapshotIDSNames(d) + + items := []fuse.Dirent{ + { + Inode: d.inode, + Name: ".", + Type: fuse.DT_Dir, + }, + { + Inode: d.root.inode, + Name: "..", + Type: fuse.DT_Dir, + }, + } + + for name := range d.names { + items = append(items, fuse.Dirent{ + Inode: fs.GenerateDynamicInode(d.inode, name), + Name: name, + Type: fuse.DT_Dir, + }) + } + + return items, nil } +// ReadDirAll returns all entries of the HostsDir. +func (d *HostsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + debug.Log("ReadDirAll()") + + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getHostsNames(d) + + items := []fuse.Dirent{ + { + Inode: d.inode, + Name: ".", + Type: fuse.DT_Dir, + }, + { + Inode: d.root.inode, + Name: "..", + Type: fuse.DT_Dir, + }, + } + + for host := range d.hosts { + items = append(items, fuse.Dirent{ + Inode: fs.GenerateDynamicInode(d.inode, host), + Name: host, + Type: fuse.DT_Dir, + }) + } + + return items, nil +} + +// ReadDirAll returns all entries of the TagsDir. +func (d *TagsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) { + debug.Log("ReadDirAll()") + + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getTagNames(d) + + items := []fuse.Dirent{ + { + Inode: d.inode, + Name: ".", + Type: fuse.DT_Dir, + }, + { + Inode: d.root.inode, + Name: "..", + Type: fuse.DT_Dir, + }, + } + + for tag := range d.tags { + items = append(items, fuse.Dirent{ + Inode: fs.GenerateDynamicInode(d.inode, tag), + Name: tag, + Type: fuse.DT_Dir, + }) + } + + return items, nil +} + +// newSnapshotLink func newSnapshotLink(ctx context.Context, root *Root, inode uint64, target string, snapshot *restic.Snapshot) (*snapshotLink, error) { return &snapshotLink{root: root, inode: inode, target: target, snapshot: snapshot}, nil } +// Readlink func (l *snapshotLink) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string, error) { return l.target, nil } +// Attr func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { a.Inode = l.inode a.Mode = os.ModeSymlink | 0777 @@ -166,24 +401,104 @@ func (l *snapshotLink) Attr(ctx context.Context, a *fuse.Attr) error { return nil } -// Lookup returns a specific entry from the root node. +// Lookup returns a specific entry from the SnapshotsDir. func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { debug.Log("Lookup(%s)", name) sn, ok := d.names[name] if !ok { + // could not find entry. Updating repository-state + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getSnapshotNames(d) + + sn, ok := d.names[name] + if ok { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) + } + if name == "latest" && d.latest != "" { - sn2, ok2 := d.names[d.latest] + sn, ok := d.names[d.latest] // internal error - if !ok2 { + if !ok { return nil, fuse.ENOENT } - return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn2) + return newSnapshotLink(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), d.latest, sn) } return nil, fuse.ENOENT } return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) } + +// Lookup returns a specific entry from the SnapshotsIDSDir. +func (d *SnapshotsIDSDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("Lookup(%s)", name) + + sn, ok := d.names[name] + if !ok { + // could not find entry. Updating repository-state + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getSnapshotIDSNames(d) + + sn, ok := d.names[name] + if ok { + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) + } + + return nil, fuse.ENOENT + } + + return newDirFromSnapshot(ctx, d.root, fs.GenerateDynamicInode(d.inode, name), sn) +} + +// Lookup returns a specific entry from the HostsDir. +func (d *HostsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("Lookup(%s)", name) + + _, ok := d.hosts[name] + if !ok { + // could not find entry. Updating repository-state + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getHostsNames(d) + + _, ok := d.hosts[name] + if ok { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil + } + + return nil, fuse.ENOENT + } + + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), "", name), nil +} + +// Lookup returns a specific entry from the TagsDir. +func (d *TagsDir) Lookup(ctx context.Context, name string) (fs.Node, error) { + debug.Log("Lookup(%s)", name) + + _, ok := d.tags[name] + if !ok { + // could not find entry. Updating repository-state + d.root.repo.LoadIndex(ctx) + d.root.snapshots = restic.FindFilteredSnapshots(ctx, d.root.repo, d.root.cfg.Host, d.root.cfg.Tags, d.root.cfg.Paths) + + getTagNames(d) + + _, ok := d.tags[name] + if ok { + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil + } + + return nil, fuse.ENOENT + } + + return NewSnapshotsDir(d.root, fs.GenerateDynamicInode(d.root.inode, name), name, ""), nil +}