From ffc6b3d887876b7e239403187af7ccc062abf69f Mon Sep 17 00:00:00 2001 From: DRON-666 <64691982+DRON-666@users.noreply.github.com> Date: Tue, 2 Feb 2021 17:43:40 +0300 Subject: [PATCH 1/3] Move PreallocateFile to fs package --- internal/{restorer => fs}/preallocate_darwin.go | 4 ++-- internal/{restorer => fs}/preallocate_linux.go | 4 ++-- internal/{restorer => fs}/preallocate_other.go | 4 ++-- internal/{restorer => fs}/preallocate_test.go | 7 +++---- internal/restorer/fileswriter.go | 3 ++- 5 files changed, 11 insertions(+), 11 deletions(-) rename internal/{restorer => fs}/preallocate_darwin.go (87%) rename internal/{restorer => fs}/preallocate_linux.go (76%) rename internal/{restorer => fs}/preallocate_other.go (73%) rename internal/{restorer => fs}/preallocate_test.go (85%) diff --git a/internal/restorer/preallocate_darwin.go b/internal/fs/preallocate_darwin.go similarity index 87% rename from internal/restorer/preallocate_darwin.go rename to internal/fs/preallocate_darwin.go index ae6e5ee3e..af46e971b 100644 --- a/internal/restorer/preallocate_darwin.go +++ b/internal/fs/preallocate_darwin.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -6,7 +6,7 @@ import ( "golang.org/x/sys/unix" ) -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { // try contiguous first fst := unix.Fstore_t{ Flags: unix.F_ALLOCATECONTIG | unix.F_ALLOCATEALL, diff --git a/internal/restorer/preallocate_linux.go b/internal/fs/preallocate_linux.go similarity index 76% rename from internal/restorer/preallocate_linux.go rename to internal/fs/preallocate_linux.go index dc73ddfe2..30b9e4644 100644 --- a/internal/restorer/preallocate_linux.go +++ b/internal/fs/preallocate_linux.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -6,7 +6,7 @@ import ( "golang.org/x/sys/unix" ) -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { if size <= 0 { return nil } diff --git a/internal/restorer/preallocate_other.go b/internal/fs/preallocate_other.go similarity index 73% rename from internal/restorer/preallocate_other.go rename to internal/fs/preallocate_other.go index f01757bf4..4fb44d421 100644 --- a/internal/restorer/preallocate_other.go +++ b/internal/fs/preallocate_other.go @@ -1,11 +1,11 @@ //go:build !linux && !darwin // +build !linux,!darwin -package restorer +package fs import "os" -func preallocateFile(wr *os.File, size int64) error { +func PreallocateFile(wr *os.File, size int64) error { // Maybe truncate can help? // Windows: This calls SetEndOfFile which preallocates space on disk return wr.Truncate(size) diff --git a/internal/restorer/preallocate_test.go b/internal/fs/preallocate_test.go similarity index 85% rename from internal/restorer/preallocate_test.go rename to internal/fs/preallocate_test.go index 0cc2b3f5d..9dabd2f36 100644 --- a/internal/restorer/preallocate_test.go +++ b/internal/fs/preallocate_test.go @@ -1,4 +1,4 @@ -package restorer +package fs import ( "os" @@ -7,7 +7,6 @@ import ( "syscall" "testing" - "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/test" ) @@ -23,7 +22,7 @@ func TestPreallocate(t *testing.T) { test.OK(t, wr.Close()) }() - err = preallocateFile(wr, i) + err = PreallocateFile(wr, i) if err == syscall.ENOTSUP { t.SkipNow() } @@ -32,7 +31,7 @@ func TestPreallocate(t *testing.T) { fi, err := wr.Stat() test.OK(t, err) - efi := fs.ExtendedStat(fi) + efi := ExtendedStat(fi) test.Assert(t, efi.Size == i || efi.Blocks > 0, "Preallocated size of %v, got size %v block %v", i, efi.Size, efi.Blocks) }) } diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 0a26101f4..589aa502a 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -6,6 +6,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/fs" ) // writes blobs to target files. @@ -72,7 +73,7 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create return nil, err } } else { - err := preallocateFile(wr.File, createSize) + err := fs.PreallocateFile(wr.File, createSize) if err != nil { // Just log the preallocate error but don't let it cause the restore process to fail. // Preallocate might return an error if the filesystem (implementation) does not From c37d587f81f27bd1c4588fe4526684b36f7a90cf Mon Sep 17 00:00:00 2001 From: DRON-666 <64691982+DRON-666@users.noreply.github.com> Date: Tue, 2 Feb 2021 17:44:40 +0300 Subject: [PATCH 2/3] Use PreallocateFile in local backend --- internal/backend/local/local.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index be332e8ad..d6bdef1e4 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -148,6 +148,13 @@ func (b *Local) Save(_ context.Context, h restic.Handle, rd restic.RewindReader) } }(f) + // preallocate disk space + if size := rd.Length(); size > 0 { + if err := fs.PreallocateFile(f, size); err != nil { + debug.Log("Failed to preallocate %v with size %v: %v", finalname, size, err) + } + } + // save data, then sync wbytes, err := io.Copy(f, rd) if err != nil { From 74e4656850f7e36cc19c12a8837674a7064f1a65 Mon Sep 17 00:00:00 2001 From: DRON-666 <64691982+DRON-666@users.noreply.github.com> Date: Tue, 2 Feb 2021 17:45:10 +0300 Subject: [PATCH 3/3] Update changelog --- changelog/unreleased/pull-3261 | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog/unreleased/pull-3261 diff --git a/changelog/unreleased/pull-3261 b/changelog/unreleased/pull-3261 new file mode 100644 index 000000000..f7073ed7b --- /dev/null +++ b/changelog/unreleased/pull-3261 @@ -0,0 +1,8 @@ +Enhancement: Reduce file fragmentation for local backend + +Before this change, local backend files could become fragmented. +Now restic will try to preallocate space for pack files to avoid +their fragmentation. + +https://github.com/restic/restic/issues/2679 +https://github.com/restic/restic/pull/3261