From fa25d6118ed8f514a6915aff50c3bd037b4bb0ac Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 7 Aug 2021 23:33:43 +0200 Subject: [PATCH] archiver: Reduce tree saver concurrency Large amount of tree savers have no obvious benefit, however they can increase the amount of (potentially large) trees kept in memory. --- internal/archiver/archiver.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 5aa509449..7d28fcb30 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -128,9 +128,12 @@ func (o Options) ApplyDefaults() Options { } if o.SaveTreeConcurrency == 0 { - // use a relatively high concurrency here, having multiple SaveTree - // workers is cheap - o.SaveTreeConcurrency = o.SaveBlobConcurrency * 20 + // can either wait for a file, wait for a tree, serialize a tree or wait for saveblob + // the last two are cpu-bound and thus mutually exclusive. + // Also allow waiting for FileReadConcurrency files, this is the maximum of FutureFiles + // which currently can be in progress. The main backup loop blocks when trying to queue + // more files to read. + o.SaveTreeConcurrency = uint(runtime.GOMAXPROCS(0)) + o.FileReadConcurrency } return o