Merge pull request #4365 from MichaelEischer/ci-rework-build-tests

CI: rework build tests

.github/workflows/tests.yml

@@ -201,27 +201,19 @@ jobs:
   cross_compile:
     strategy:
 
-      # ATTENTION: the list of architectures must be in sync with helpers/build-release-binaries/main.go!
       matrix:
         # run cross-compile in three batches parallel so the overall tests run faster
-        targets:
+        subset:
-          - "linux/386 linux/amd64 linux/arm linux/arm64 linux/ppc64le linux/mips linux/mipsle linux/mips64 linux/mips64le linux/riscv64 linux/s390x"
+          - "0/3"
-
+          - "1/3"
-          - "openbsd/386 openbsd/amd64 \
+          - "2/3"
-            freebsd/386 freebsd/amd64 freebsd/arm \
-            aix/ppc64 \
-            darwin/amd64 darwin/arm64"
-
-          - "netbsd/386 netbsd/amd64 \
-            windows/386 windows/amd64 \
-            solaris/amd64"
 
     env:
       GOPROXY: https://proxy.golang.org
 
     runs-on: ubuntu-latest
 
-    name: Cross Compile for ${{ matrix.targets }}
+    name: Cross Compile for subset ${{ matrix.subset }}
 
     steps:
       - name: Set up Go ${{ env.latest_go }}
@@ -229,21 +221,14 @@ jobs:
         with:
           go-version: ${{ env.latest_go }}
 
-      - name: Install gox
-        run: |
-          go install github.com/mitchellh/gox@latest
-
       - name: Check out code
         uses: actions/checkout@v3
 
-      - name: Cross-compile with gox for ${{ matrix.targets }}
+      - name: Cross-compile for subset ${{ matrix.subset }}
-        env:
-          GOFLAGS: "-trimpath"
-          GOX_ARCHS: "${{ matrix.targets }}"
         run: |
-          mkdir build-output
+          mkdir build-output build-output-debug
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}" ./cmd/restic
+          go run ./helpers/build-release-binaries/main.go -o build-output -s . --platform-subset ${{ matrix.subset }}
-          gox -parallel 2 -verbose -osarch "$GOX_ARCHS" -tags debug -output "build-output/{{.Dir}}_{{.OS}}_{{.Arch}}_debug" ./cmd/restic
+          go run ./helpers/build-release-binaries/main.go -o build-output-debug -s . --platform-subset ${{ matrix.subset }} --tags debug
 
   lint:
     name: lint

changelog/unreleased/pull-4365

@@ -0,0 +1,6 @@
+Change: Building restic on AIX is temporarily unsupported
+
+As the current version of the library used for the Azure backend does not
+compile on AIX, there are currently no restic builds available for AIX.
+
+https://github.com/restic/restic/pull/4365

helpers/build-release-binaries/main.go

@@ -1,11 +1,14 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -14,16 +17,20 @@ import (
 )
 
 var opts = struct {
-	Verbose   bool
+	Verbose        bool
-	SourceDir string
+	SourceDir      string
-	OutputDir string
+	OutputDir      string
-	Version   string
+	Tags           string
+	PlatformSubset string
+	Version        string
 }{}
 
 func init() {
 	pflag.BoolVarP(&opts.Verbose, "verbose", "v", false, "be verbose")
 	pflag.StringVarP(&opts.SourceDir, "source", "s", "/restic", "path to the source code `directory`")
 	pflag.StringVarP(&opts.OutputDir, "output", "o", "/output", "path to the output `directory`")
+	pflag.StringVar(&opts.Tags, "tags", "", "additional build `tags`")
+	pflag.StringVar(&opts.PlatformSubset, "platform-subset", "", "specify `n/t` to only build this subset")
 	pflag.StringVar(&opts.Version, "version", "", "use `x.y.z` as the version for output files")
 	pflag.Parse()
 }
@@ -95,10 +102,15 @@ func build(sourceDir, outputDir, goos, goarch string) (filename string) {
 	}
 	outputFile := filepath.Join(outputDir, filename)
 
+	tags := "selfupdate"
+	if opts.Tags != "" {
+		tags += "," + opts.Tags
+	}
+
 	c := exec.Command("go", "build",
 		"-o", outputFile,
 		"-ldflags", "-s -w",
-		"-tags", "selfupdate",
+		"-tags", tags,
 		"./cmd/restic",
 	)
 	c.Stdout = os.Stdout
@@ -220,9 +232,7 @@ func buildTargets(sourceDir, outputDir string, targets map[string][]string) {
 	msg("build finished in %.3fs", time.Since(start).Seconds())
 }
 
-// ATTENTION: the list of architectures must be in sync with .github/workflows/tests.yml!
 var defaultBuildTargets = map[string][]string{
-	"aix":     {"ppc64"},
 	"darwin":  {"amd64", "arm64"},
 	"freebsd": {"386", "amd64", "arm"},
 	"linux":   {"386", "amd64", "arm", "arm64", "ppc64le", "mips", "mipsle", "mips64", "mips64le", "riscv64", "s390x"},
@@ -244,15 +254,69 @@ func downloadModules(sourceDir string) {
 	}
 }
 
+func selectSubset(subset string, target map[string][]string) (map[string][]string, error) {
+	t, n, _ := strings.Cut(subset, "/")
+	part, err := strconv.ParseInt(t, 10, 8)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse platform subset %q", subset)
+	}
+	total, err := strconv.ParseInt(n, 10, 8)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse platform subset %q", subset)
+	}
+	if total < 0 || part < 0 {
+		return nil, errors.New("platform subset out of range")
+	}
+	if part >= total {
+		return nil, errors.New("t must be in 0 <= t < n")
+	}
+
+	// flatten platform list
+	platforms := []string{}
+	for os, archs := range target {
+		for _, arch := range archs {
+			platforms = append(platforms, os+"/"+arch)
+		}
+	}
+	sort.Strings(platforms)
+
+	// select subset
+	lower := len(platforms) * int(part) / int(total)
+	upper := len(platforms) * int(part+1) / int(total)
+	platforms = platforms[lower:upper]
+
+	return buildPlatformList(platforms), nil
+}
+
+func buildPlatformList(platforms []string) map[string][]string {
+	fmt.Printf("Building for %v\n", platforms)
+
+	targets := make(map[string][]string)
+	for _, platform := range platforms {
+		os, arch, _ := strings.Cut(platform, "/")
+		targets[os] = append(targets[os], arch)
+	}
+	return targets
+}
+
 func main() {
 	if len(pflag.Args()) != 0 {
 		die("USAGE: build-release-binaries [OPTIONS]")
 	}
 
+	targets := defaultBuildTargets
+	if opts.PlatformSubset != "" {
+		var err error
+		targets, err = selectSubset(opts.PlatformSubset, targets)
+		if err != nil {
+			die("%s", err)
+		}
+	}
+
 	sourceDir := abs(opts.SourceDir)
 	outputDir := abs(opts.OutputDir)
 	mkdir(outputDir)
 
 	downloadModules(sourceDir)
-	buildTargets(sourceDir, outputDir, defaultBuildTargets)
+	buildTargets(sourceDir, outputDir, targets)
 }