From e530d422a0cde51291cd17b528b4643504dcc0b2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Aug 2022 12:09:42 +0200 Subject: [PATCH 1/4] helper: don't setup cmd paths twice --- helpers/build-release-binaries/main.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index 0c1e6527c..f81498bdf 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -103,7 +103,6 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { ) c.Stdout = os.Stdout c.Stderr = os.Stderr - c.Dir = sourceDir verbose("run %v %v in %v", "go", c.Args, c.Dir) @@ -151,11 +150,9 @@ func compress(goos, inputDir, filename string) (outputFile string) { case "windows": outputFile = strings.TrimSuffix(filename, ".exe") + ".zip" c = exec.Command("zip", "-q", "-X", outputFile, filename) - c.Dir = inputDir default: outputFile = filename + ".bz2" c = exec.Command("bzip2", filename) - c.Dir = inputDir } rm(filepath.Join(inputDir, outputFile)) From ed94678820af7fbe53dd12a56d97f3d760ccaf2d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Aug 2022 12:10:29 +0200 Subject: [PATCH 2/4] helper: cleanups --- helpers/build-release-binaries/main.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index f81498bdf..b176f8b8c 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -103,15 +103,13 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) { ) c.Stdout = os.Stdout c.Stderr = os.Stderr - - verbose("run %v %v in %v", "go", c.Args, c.Dir) - c.Dir = sourceDir c.Env = append(os.Environ(), "CGO_ENABLED=0", "GOOS="+goos, "GOARCH="+goarch, ) + verbose("run %v %v in %v", "go", c.Args, c.Dir) err := c.Run() if err != nil { @@ -160,7 +158,6 @@ func compress(goos, inputDir, filename string) (outputFile string) { c.Stdout = os.Stdout c.Stderr = os.Stderr c.Dir = inputDir - verbose("run %v %v in %v", "go", c.Args, c.Dir) err := c.Run() From 7f0929e5198bb9439a4304085979a7e2e46c6a2d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Aug 2022 12:10:48 +0200 Subject: [PATCH 3/4] helper: Reduce number of parallel builds a bit The go compiler is already parallelized. The high concurrency caused my podman container to hit a resource limit. --- helpers/build-release-binaries/main.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index b176f8b8c..df6c4d2bf 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -182,14 +182,19 @@ func buildForTarget(sourceDir, outputDir, goos, goarch string) (filename string) func buildTargets(sourceDir, outputDir string, targets map[string][]string) { start := time.Now() - msg("building with %d workers", runtime.NumCPU()) + // the go compiler is already parallelized, thus reduce the concurrency a bit + workers := runtime.GOMAXPROCS(0) / 4 + if workers < 1 { + workers = 1 + } + msg("building with %d workers", workers) type Job struct{ GOOS, GOARCH string } var wg errgroup.Group ch := make(chan Job) - for i := 0; i < runtime.NumCPU(); i++ { + for i := 0; i < workers; i++ { wg.Go(func() error { for job := range ch { start := time.Now() From c3374b3ea5ab33d6bfdf25d95c85a98ef5008b09 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Aug 2022 12:11:51 +0200 Subject: [PATCH 4/4] helper: download modules as first step There's no use in running that step in parallel. --- helpers/build-release-binaries/main.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/helpers/build-release-binaries/main.go b/helpers/build-release-binaries/main.go index df6c4d2bf..1662ada0b 100644 --- a/helpers/build-release-binaries/main.go +++ b/helpers/build-release-binaries/main.go @@ -232,6 +232,18 @@ var defaultBuildTargets = map[string][]string{ "solaris": {"amd64"}, } +func downloadModules(sourceDir string) { + c := exec.Command("go", "mod", "download") + c.Stdout = os.Stdout + c.Stderr = os.Stderr + c.Dir = sourceDir + + err := c.Run() + if err != nil { + die("error downloading modules: %v", err) + } +} + func main() { if len(pflag.Args()) != 0 { die("USAGE: build-release-binaries [OPTIONS]") @@ -241,5 +253,6 @@ func main() { outputDir := abs(opts.OutputDir) mkdir(outputDir) + downloadModules(sourceDir) buildTargets(sourceDir, outputDir, defaultBuildTargets) }