From feb664620a19fea606d77e217e438eb488b38508 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 28 Mar 2016 15:26:04 +0200 Subject: [PATCH] Use fadvise() to not cache the content of files read --- src/restic/archiver.go | 7 ++-- src/restic/fs/doc.go | 3 ++ src/restic/fs/file.go | 18 ++++++++++ src/restic/fs/file_all.go | 20 +++++++++++ src/restic/fs/file_linux.go | 69 +++++++++++++++++++++++++++++++++++++ src/restic/node_darwin.go | 9 +---- src/restic/node_freebsd.go | 9 +---- src/restic/node_linux.go | 8 ----- src/restic/node_openbsd.go | 13 +------ src/restic/node_windows.go | 5 --- 10 files changed, 117 insertions(+), 44 deletions(-) create mode 100644 src/restic/fs/doc.go create mode 100644 src/restic/fs/file.go create mode 100644 src/restic/fs/file_all.go create mode 100644 src/restic/fs/file_linux.go diff --git a/src/restic/archiver.go b/src/restic/archiver.go index 9ff7ca10b..9f9298f83 100644 --- a/src/restic/archiver.go +++ b/src/restic/archiver.go @@ -13,6 +13,7 @@ import ( "restic/backend" "restic/debug" + "restic/fs" "restic/pack" "restic/pipe" "restic/repository" @@ -126,7 +127,7 @@ func (arch *Archiver) SaveTreeJSON(item interface{}) (backend.ID, error) { return arch.repo.SaveJSON(pack.Tree, item) } -func (arch *Archiver) reloadFileIfChanged(node *Node, file *os.File) (*Node, error) { +func (arch *Archiver) reloadFileIfChanged(node *Node, file fs.File) (*Node, error) { fi, err := file.Stat() if err != nil { return nil, err @@ -155,7 +156,7 @@ type saveResult struct { bytes uint64 } -func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}, file *os.File, resultChannel chan<- saveResult) { +func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}, file fs.File, resultChannel chan<- saveResult) { defer freeBuf(chunk.Data) id := backend.Hash(chunk.Data) @@ -209,7 +210,7 @@ func updateNodeContent(node *Node, results []saveResult) error { // SaveFile stores the content of the file on the backend as a Blob by calling // Save for each chunk. func (arch *Archiver) SaveFile(p *Progress, node *Node) error { - file, err := node.OpenForReading() + file, err := fs.Open(node.path) defer file.Close() if err != nil { return err diff --git a/src/restic/fs/doc.go b/src/restic/fs/doc.go new file mode 100644 index 000000000..29bfa6661 --- /dev/null +++ b/src/restic/fs/doc.go @@ -0,0 +1,3 @@ +// Package fs implements an OS independend abstraction of a file system +// suitable for backup purposes. +package fs diff --git a/src/restic/fs/file.go b/src/restic/fs/file.go new file mode 100644 index 000000000..84c003c5d --- /dev/null +++ b/src/restic/fs/file.go @@ -0,0 +1,18 @@ +package fs + +import ( + "io" + "os" +) + +// File is an open file on a file system. +type File interface { + io.Reader + io.Writer + io.Closer + + Stat() (os.FileInfo, error) + + // ClearCache removes the file's content from the OS cache. + ClearCache() error +} diff --git a/src/restic/fs/file_all.go b/src/restic/fs/file_all.go new file mode 100644 index 000000000..b3ee2ba95 --- /dev/null +++ b/src/restic/fs/file_all.go @@ -0,0 +1,20 @@ +// +build !linux + +package fs + +import "os" + +// Open opens a file for reading. +func Open(name string) (File, error) { + f, err := os.OpenFile(name, os.O_RDONLY, 0) + return osFile{File: f}, err +} + +// osFile wraps an *os.File and adds a no-op ClearCache() method. +type osFile struct { + *os.File +} + +func (osFile) ClearCache() error { + return nil +} diff --git a/src/restic/fs/file_linux.go b/src/restic/fs/file_linux.go new file mode 100644 index 000000000..d79968be3 --- /dev/null +++ b/src/restic/fs/file_linux.go @@ -0,0 +1,69 @@ +package fs + +import ( + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +// Open opens a file for reading, without updating the atime and without caching data on read. +func Open(name string) (File, error) { + file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_NOATIME, 0) + if os.IsPermission(err) { + file, err = os.OpenFile(name, os.O_RDONLY, 0) + } + return &nonCachingFile{File: file}, err +} + +// osFile wraps an *os.File and adds a no-op ClearCache() method. +type osFile struct { + *os.File +} + +func (osFile) ClearCache() error { + return nil +} + +// these constants should've been defined in x/sys/unix, but somehow aren't. +const ( + _POSIX_FADV_NORMAL = iota + _POSIX_FADV_RANDOM + _POSIX_FADV_SEQUENTIAL + _POSIX_FADV_WILLNEED + _POSIX_FADV_DONTNEED + _POSIX_FADV_NOREUSE +) + +// nonCachingFile wraps an *os.File and calls fadvise() to instantly forget +// data that has been read or written. +type nonCachingFile struct { + *os.File + readOffset int64 +} + +func (f *nonCachingFile) Read(p []byte) (int, error) { + n, err := f.File.Read(p) + + if n > 0 { + ferr := unix.Fadvise(int(f.File.Fd()), f.readOffset, int64(n), _POSIX_FADV_DONTNEED) + + f.readOffset += int64(n) + + if err == nil { + err = ferr + } + } + + return n, err +} + +func (f *nonCachingFile) ClearCache() error { + err := f.File.Sync() + + if err != nil { + return err + } + + return unix.Fadvise(int(f.File.Fd()), 0, 0, _POSIX_FADV_DONTNEED) +} diff --git a/src/restic/node_darwin.go b/src/restic/node_darwin.go index adc980295..a3f97096d 100644 --- a/src/restic/node_darwin.go +++ b/src/restic/node_darwin.go @@ -1,13 +1,6 @@ package restic -import ( - "os" - "syscall" -) - -func (node *Node) OpenForReading() (*os.File, error) { - return os.Open(node.path) -} +import "syscall" func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil diff --git a/src/restic/node_freebsd.go b/src/restic/node_freebsd.go index 67bcdf3e9..a3f97096d 100644 --- a/src/restic/node_freebsd.go +++ b/src/restic/node_freebsd.go @@ -1,13 +1,6 @@ package restic -import ( - "os" - "syscall" -) - -func (node *Node) OpenForReading() (*os.File, error) { - return os.OpenFile(node.path, os.O_RDONLY, 0) -} +import "syscall" func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil diff --git a/src/restic/node_linux.go b/src/restic/node_linux.go index 2304c13d5..c6b50ee38 100644 --- a/src/restic/node_linux.go +++ b/src/restic/node_linux.go @@ -9,14 +9,6 @@ import ( "github.com/juju/errors" ) -func (node *Node) OpenForReading() (*os.File, error) { - file, err := os.OpenFile(node.path, os.O_RDONLY|syscall.O_NOATIME, 0) - if os.IsPermission(err) { - return os.OpenFile(node.path, os.O_RDONLY, 0) - } - return file, err -} - func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { dir, err := os.Open(filepath.Dir(path)) defer dir.Close() diff --git a/src/restic/node_openbsd.go b/src/restic/node_openbsd.go index feafe307e..4c7779835 100644 --- a/src/restic/node_openbsd.go +++ b/src/restic/node_openbsd.go @@ -1,17 +1,6 @@ package restic -import ( - "os" - "syscall" -) - -func (node *Node) OpenForReading() (*os.File, error) { - file, err := os.OpenFile(node.path, os.O_RDONLY, 0) - if os.IsPermission(err) { - return os.OpenFile(node.path, os.O_RDONLY, 0) - } - return file, err -} +import "syscall" func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { return nil diff --git a/src/restic/node_windows.go b/src/restic/node_windows.go index 5a5a9dba3..c238cf87c 100644 --- a/src/restic/node_windows.go +++ b/src/restic/node_windows.go @@ -2,14 +2,9 @@ package restic import ( "errors" - "os" "syscall" ) -func (node *Node) OpenForReading() (*os.File, error) { - return os.OpenFile(node.path, os.O_RDONLY, 0) -} - // mknod() creates a filesystem node (file, device // special file, or named pipe) named pathname, with attributes // specified by mode and dev.