ci: investigate hosted test bottlenecks (Linux and Windows)

.github/workflows/test-profile.yml

@@ -0,0 +1,103 @@
+name: Hosted test profile
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/test-profile.yml
+      - scripts/ci-test-profile.sh
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+  profile:
+    # Investigation only; do not add this expensive matrix to the normal gate.
+    if: github.head_ref == 'ci/profile-hosted-tests' || (github.event_name == 'workflow_dispatch' && github.ref != 'refs/heads/main')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        cache: [restored, cold]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    defaults:
+      run:
+        shell: bash
+    env:
+      PROFILE_CACHE: ${{ matrix.cache }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: $/.github/actions/setup-go
+        with:
+          cache-name: ${{ matrix.os == 'windows-latest' && 'test-windows' || 'test-linux' }}
+          # Unique misses leave shared caches intact; PRs never save caches.
+          cache-version: ${{ matrix.cache == 'cold' && format('profile-{0}-{1}', github.run_id, github.run_attempt) || '2' }}
+          save: 'false'
+          task: 'true'
+
+      - name: Exclude build paths from Defender
+        if: runner.os == 'Windows'
+        continue-on-error: true
+        shell: pwsh
+        run: |
+          $paths = @($env:GITHUB_WORKSPACE, $env:RUNNER_TEMP, (go env GOROOT), (go env GOPATH), (go env GOCACHE), (go env GOMODCACHE))
+          Add-MpPreference -ExclusionPath $paths
+          Get-MpPreference | Select-Object -ExpandProperty ExclusionPath
+
+      - name: Runner and cache environment
+        run: bash scripts/ci-test-profile.sh environment
+
+      - name: Download modules
+        run: bash scripts/ci-test-profile.sh download
+
+      - name: Build (Linux CI only)
+        if: runner.os == 'Linux'
+        run: bash scripts/ci-test-profile.sh build
+
+      - name: Compile all tests without executing
+        run: bash scripts/ci-test-profile.sh compile
+
+      - name: Execute all tests (uncached results, first pass)
+        if: runner.os == 'Linux'
+        run: bash scripts/ci-test-profile.sh execute-1
+
+      - name: Execute all tests (uncached results, second pass)
+        if: runner.os == 'Linux' && (success() || failure())
+        run: bash scripts/ci-test-profile.sh execute-2
+
+      - name: Execute Windows tests through the normal CI launcher
+        if: runner.os == 'Windows' && (success() || failure())
+        shell: pwsh
+        run: |
+          $dir = Join-Path $env:RUNNER_TEMP 'ci-test-profile'
+          $failed = $false
+          foreach ($pass in 1..2) {
+            "## Windows uncached pass $pass (PowerShell + Task)" >> $env:GITHUB_STEP_SUMMARY
+            $timer = [Diagnostics.Stopwatch]::StartNew()
+            task test-ci -- -count=1
+            $status = $LASTEXITCODE
+            $timer.Stop()
+            $seconds = [int]$timer.Elapsed.TotalSeconds
+            "execute-$pass`t$seconds`t$status" >> (Join-Path $dir 'phases.tsv')
+            Move-Item (Join-Path $env:RUNNER_TEMP 'go-test.json') (Join-Path $dir "execute-$pass.json") -Force
+            "Wall time: ${seconds}s; exit status: $status" >> $env:GITHUB_STEP_SUMMARY
+            if ($status -ne 0) { $failed = $true }
+          }
+          if ($failed) { exit 1 }
+
+      - name: Upload measurements
+        if: always()
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: test-profile-${{ matrix.os }}-${{ matrix.cache }}-${{ github.run_attempt }}
+          path: ${{ runner.temp }}/ci-test-profile
+          retention-days: 14
+          if-no-files-found: warn

Taskfile.yml

@@ -111,7 +111,7 @@ tasks:
     desc: Run tests with CI timing reports
     dotenv: [.env.test]
     cmds:
-      - set -o pipefail; go test -json ./... | go run ./scripts/gotestreport -out "${RUNNER_TEMP:-/tmp}/go-test.json"
+      - set -o pipefail; go test -json {{.CLI_ARGS}} ./... | go run ./scripts/gotestreport -out "${RUNNER_TEMP:-/tmp}/go-test.json"
 
   test-race:
     desc: Run tests with the race detector and shuffled order (slow; CI runs it on main only)

scripts/ci-test-profile.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUNNER_TEMP:?Run this diagnostic on a hosted CI runner}"
+export PROFILE_DIR="$RUNNER_TEMP/ci-test-profile"
+mkdir -p "$PROFILE_DIR"
+
+# Match Taskfile test environments without passing live provider credentials.
+set -a
+# shellcheck disable=SC1091
+source .env.test
+set +a
+
+time_phase() {
+  local phase="$1" start end status=0
+  shift
+  {
+    printf '## Phase: %s\n\n' "$phase"
+    printf '| Phase | Wall time | Exit status |\n| --- | ---: | ---: |\n'
+  } >> "$GITHUB_STEP_SUMMARY"
+  start=$(date +%s)
+  if "$@"; then
+    status=0
+  else
+    status=$?
+  fi
+  end=$(date +%s)
+  printf '%s\t%s\t%s\n' "$phase" "$((end - start))" "$status" >> "$PROFILE_DIR/phases.tsv"
+  printf '| %s | %ss | %s |\n' "$phase" "$((end - start))" "$status" >> "$GITHUB_STEP_SUMMARY"
+  return "$status"
+}
+
+execute_tests() {
+  go test -count=1 -json ./... | "$PROFILE_DIR/gotestreport" -out "$PROFILE_DIR/$PROFILE_PASS.json"
+}
+
+case "${1:?Specify a profiling phase}" in
+  environment)
+    {
+      printf 'Commit: %s\nCache scenario: %s\nRunner: %s/%s\n' "$GITHUB_SHA" "$PROFILE_CACHE" "$RUNNER_OS" "$RUNNER_ARCH"
+      printf 'Image: %s %s\n' "${ImageOS:-unknown}" "${ImageVersion:-unknown}"
+      printf 'Logical CPUs: '
+      nproc
+      go version
+      go env -json GOOS GOARCH GOFLAGS GOTOOLCHAIN GOROOT GOPATH GOCACHE GOMODCACHE CGO_ENABLED
+      printf 'GOMAXPROCS override: %s\n' "${GOMAXPROCS:-unset (Go default)}"
+      printf 'HOME=%s\nUSERPROFILE=%s\nTEMP=%s\nTMP=%s\n' "${HOME:-}" "${USERPROFILE:-}" "${TEMP:-}" "${TMP:-}"
+      df -h . "$RUNNER_TEMP"
+      if [[ "$RUNNER_OS" == Linux ]]; then
+        cat /proc/meminfo
+        lscpu
+        for path in /sys/fs/cgroup/cpu.max /sys/fs/cgroup/memory.max; do
+          if [[ -f "$path" ]]; then printf '%s: ' "$path"; cat "$path"; fi
+        done
+      else
+        pwsh -NoProfile -Command 'Get-CimInstance Win32_ComputerSystem | Select-Object NumberOfLogicalProcessors,TotalPhysicalMemory; Get-CimInstance Win32_Processor | Select-Object Name,NumberOfCores,NumberOfLogicalProcessors'
+      fi
+    } | tee "$PROFILE_DIR/environment.txt"
+    {
+      printf '## Hosted test profile: %s / %s\n\n' "$RUNNER_OS" "$PROFILE_CACHE"
+      printf 'Both test passes use -count=1 after a compile-only prewarm; go test still includes linking, vet, and test setup. Module download is measured separately.\n\n'
+    } >> "$GITHUB_STEP_SUMMARY"
+    ;;
+  download)
+    time_phase download go mod download
+    ;;
+  build)
+    time_phase build bash scripts/build.sh
+    ;;
+  compile)
+    output=/dev/null
+    if [[ "$RUNNER_OS" == Windows ]]; then output=NUL; fi
+    time_phase compile go test -c -o "$output" ./...
+    go build -o "$PROFILE_DIR/gotestreport" ./scripts/gotestreport
+    ;;
+  execute-1|execute-2)
+    export PROFILE_PASS="$1"
+    time_phase "$1" execute_tests
+    ;;
+  *)
+    printf 'Unknown phase: %s\n' "$1" >&2
+    exit 2
+    ;;
+esac