From 5f64e26f4c2cd6afb25d8689428b260b2a330025 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Fri, 16 Oct 2026 01:12:11 +0200 Subject: [PATCH] ci: cut the per-push job fan-out that queues CI behind the runner limit CI jobs spent ~13,000 min queued against ~4,600 min running over the last week: one CI run starts ~17-25 jobs, about the org's concurrency limit. - CodeQL: on pull requests, analyze Rust, Go, Python and Actions only when their paths change. Main pushes and the weekly schedule still analyze every language. Removes up to four jobs (Rust alone is ~6 min) from most PR pushes. - Test lanes: refit the default per-file estimate to measured cold runs (0.6s, was 1s) and let Bun suites pack into the small Vitest lanes, run in their own Turbo invocation without Vitest flags. 12 lanes become 10; the dropped two did 2s and 7s of test work each. --- .github/scripts/plan-tests.py | 22 +++++++---- .github/scripts/run-tests.py | 24 +++++++++--- .github/scripts/test_plan_tests.py | 47 +++++++++++++++-------- .github/workflows/codeql.yml | 60 +++++++++++++++++++++++------- docs/ci-performance.md | 15 +++++--- 5 files changed, 120 insertions(+), 48 deletions(-) diff --git a/.github/scripts/plan-tests.py b/.github/scripts/plan-tests.py index 67237490fe..89b8c97345 100644 --- a/.github/scripts/plan-tests.py +++ b/.github/scripts/plan-tests.py @@ -9,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2] # Vitest seconds/file at two workers, measured on CI run 35725366619 (Duration -# lines, excluding dependency builds). Unknown suites get the conservative default. +# lines, excluding dependency builds). Unknown suites get the default, fitted to the +# slowest cold small-suite lanes over a week of CI (2026-10): small-4, 105 files +# across 8 suites, never ran longer than 86s; the old 1s/file put it at 137s. SECONDS_PER_FILE = { "@maple/api": 2.3, "@maple/backend": 1.6, @@ -19,7 +21,7 @@ "@maple/ui": 0.45, "@maple/domain": 0.4, } -DEFAULT_SECONDS_PER_FILE = 1 +DEFAULT_SECONDS_PER_FILE = 0.6 # Every job pays ~30s of checkout/mise/install plus Turbo dependency builds before # a test runs, and the org runs ~16 jobs at once, so the matrix queued in waves. # Fewer, fuller lanes beat many short ones on both wall clock and runner minutes. @@ -72,7 +74,7 @@ def discover(root=ROOT): def pack(suites, prefix, args): groups = [] - for name, weight in sorted(suites, key=lambda item: (-item[1], item[0])): + for name, weight, vitest in sorted(suites, key=lambda item: (-item[1], item[0])): group = next( (g for g in groups if g["estimated-seconds"] + weight <= TARGET_SECONDS), None, @@ -82,28 +84,32 @@ def pack(suites, prefix, args): "name": f"{prefix}-{len(groups) + 1}", "filters": [], "args": list(args), + "bun-filters": [], "estimated-seconds": 0, } groups.append(group) group["filters"].append(name) + if not vitest: + group["bun-filters"].append(name) group["estimated-seconds"] += weight return groups def plan(suites): - lanes, small, bun = [], [], [] + lanes, small = [], [] for suite in sorted(suites, key=lambda s: s["name"]): name = suite["name"] weight = max(1, suite["files"]) * SECONDS_PER_FILE.get( name, DEFAULT_SECONDS_PER_FILE ) if not suite["vitest"]: - # Bun suites share lanes with each other, never with Vitest flags. - bun.append((name, weight + STARTUP_SECONDS)) + # Bun suites pack with the small Vitest suites; run-tests.py gives them + # their own Turbo invocation so they never receive Vitest flags. + small.append((name, weight + STARTUP_SECONDS, False)) continue shards = math.ceil(weight / (TARGET_SECONDS - STARTUP_SECONDS)) if shards == 1: - small.append((name, weight + STARTUP_SECONDS)) + small.append((name, weight + STARTUP_SECONDS, True)) else: for index in range(1, shards + 1): lanes.append( @@ -111,13 +117,13 @@ def plan(suites): "name": f"{name}-{index}-of-{shards}", "filters": [name], "args": [f"--shard={index}/{shards}", "--maxWorkers=2"], + "bun-filters": [], "estimated-seconds": weight / shards + STARTUP_SECONDS, } ) # First-fit decreasing packs small suites without ever running two test # runners on the same machine at once (Turbo concurrency=1). lanes.extend(pack(small, "small", ["--maxWorkers=2"])) - lanes.extend(pack(bun, "bun", [])) browser_workspaces = { suite["name"]: suite.get("browser-workspace", "") for suite in suites } diff --git a/.github/scripts/run-tests.py b/.github/scripts/run-tests.py index 44c3547933..e552b23b20 100644 --- a/.github/scripts/run-tests.py +++ b/.github/scripts/run-tests.py @@ -7,12 +7,24 @@ from pathlib import Path lane = json.loads(os.environ["TEST_LANE"]) -command = ["bun", "turbo", "test", "--concurrency=1"] -command.extend(f"--filter={name}" for name in lane["filters"]) -if lane["args"]: - command.extend(["--", *lane["args"]]) +bun = lane.get("bun-filters", []) +vitest = [name for name in lane["filters"] if name not in bun] +base = ["bun", "turbo", "test", "--concurrency=1"] +commands = [] +if vitest: + command = base + [f"--filter={name}" for name in vitest] + if lane["args"]: + command.extend(["--", *lane["args"]]) + commands.append(command) +# Bun suites share the lane but not its Vitest flags, which `bun test` rejects. +if bun: + commands.append(base + [f"--filter={name}" for name in bun]) start = time.monotonic() -result = subprocess.run(command, check=False) +# Both invocations run even after one fails, so a red lane reports every failure. +returncode = 0 +for command in commands: + result = subprocess.run(command, check=False) + returncode = returncode or result.returncode elapsed = time.monotonic() - start # Includes cold dependency builds. The planner targets 100s, leaving ample room # for setup and runner variance. A single slow file must be fixed/split, not @@ -29,4 +41,4 @@ print( "::error::Test lane exceeded 240s. Inspect slow files and update the CI sizing estimates; see docs/ci-performance.md." ) -raise SystemExit(result.returncode or int(elapsed > budget)) +raise SystemExit(returncode or int(elapsed > budget)) diff --git a/.github/scripts/test_plan_tests.py b/.github/scripts/test_plan_tests.py index 4c04c1f205..21747b0cdc 100644 --- a/.github/scripts/test_plan_tests.py +++ b/.github/scripts/test_plan_tests.py @@ -40,7 +40,7 @@ def assert_coverage(self, suites, matrix): ) ) else: - self.assertEqual(lanes[0]["args"], []) + self.assertIn(suite["name"], lanes[0]["bun-filters"]) self.assertEqual( {s["name"] for s in suites}, {name for lane in matrix["include"] for name in lane["filters"]}, @@ -68,7 +68,7 @@ def test_new_workspace_and_order_independence(self): self.assert_coverage(suites, planner.plan(suites)) self.assertEqual(planner.plan(suites), planner.plan(list(reversed(suites)))) - def test_bun_suites_share_a_lane_without_vitest_flags(self): + def test_bun_suites_share_small_lanes_but_are_marked(self): suites = [ {"name": "bun-a", "files": 10, "vitest": False}, {"name": "bun-b", "files": 1, "vitest": False}, @@ -76,9 +76,9 @@ def test_bun_suites_share_a_lane_without_vitest_flags(self): ] lanes = planner.plan(suites)["include"] self.assert_coverage(suites, {"include": lanes}) - bun = [lane for lane in lanes if "bun-a" in lane["filters"]] - self.assertEqual(bun[0]["filters"], ["bun-a", "bun-b"]) - self.assertEqual(bun[0]["args"], []) + self.assertEqual(len(lanes), 1) + self.assertEqual(sorted(lanes[0]["filters"]), ["bun-a", "bun-b", "vitest-a"]) + self.assertEqual(sorted(lanes[0]["bun-filters"]), ["bun-a", "bun-b"]) def test_matrix_limit_is_explicit(self): with self.assertRaisesRegex(ValueError, "256"): @@ -111,7 +111,7 @@ def test_browser_install_is_selected_for_every_affected_lane(self): class TestRunner(unittest.TestCase): - def run_lane(self, args, returncode=0, elapsed=10): + def run_lane(self, args, returncode=0, elapsed=10, bun_filters=()): import json import os import runpy @@ -122,6 +122,7 @@ def run_lane(self, args, returncode=0, elapsed=10): "name": "fixture", "filters": ["@fixture/one", "@fixture/two"], "args": args, + "bun-filters": list(bun_filters), } with ( patch.dict(os.environ, {"TEST_LANE": json.dumps(lane)}, clear=True), @@ -135,14 +136,14 @@ def run_lane(self, args, returncode=0, elapsed=10): runpy.run_path( str(Path(__file__).with_name("run-tests.py")), run_name="__main__" ) - return result.exception.code, run.call_args.args[0] + return result.exception.code, [call.args[0] for call in run.call_args_list] def test_vitest_arguments_and_failure_propagation(self): - code, command = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7) + code, commands = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7) self.assertEqual(code, 7) self.assertEqual( - command, - [ + commands, + [[ "bun", "turbo", "test", @@ -152,13 +153,29 @@ def test_vitest_arguments_and_failure_propagation(self): "--", "--shard=2/5", "--maxWorkers=2", - ], + ]], ) - def test_bun_receives_no_vitest_arguments(self): - code, command = self.run_lane([]) - self.assertEqual(code, 0) - self.assertNotIn("--", command) + def test_bun_suites_in_a_mixed_lane_get_no_vitest_arguments(self): + code, commands = self.run_lane( + ["--maxWorkers=2"], returncode=3, bun_filters=["@fixture/two"] + ) + self.assertEqual(code, 3) + self.assertEqual( + commands, + [ + [ + "bun", + "turbo", + "test", + "--concurrency=1", + "--filter=@fixture/one", + "--", + "--maxWorkers=2", + ], + ["bun", "turbo", "test", "--concurrency=1", "--filter=@fixture/two"], + ], + ) def test_slow_passing_suite_fails_budget(self): code, _ = self.run_lane([], elapsed=241) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index cbbfeac3a0..311ce769fc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,8 +3,14 @@ name: CodeQL # Replaces GitHub's default code-scanning setup, which cannot be path-filtered: # it ran the Swift analysis on a macOS runner (~10x Linux billing, 20-30 min) # for every PR, including the large majority that never touch apps/ios. Swift -# now runs only when Swift could have changed, plus on the weekly schedule; the -# other languages are cheap and keep running on every PR. +# now runs only when Swift could have changed, plus on the weekly schedule. +# +# The same goes for Rust (~6 min, 22 files in apps/ingest), Go, Python and +# Actions on pull requests: they start alongside CI and took four of the org's +# runner slots on every push for code most PRs never touch. A PR analyzes them +# only when their paths change; main pushes and the schedule analyze them all, +# so the base-branch results PRs compare against stay current. JS/TS runs on +# every event. # # Categories match default setup's (`/language:`), so alert history # carries over. Default setup must stay disabled (repo Settings → Code security) @@ -32,6 +38,7 @@ jobs: timeout-minutes: 5 outputs: swift: ${{ steps.filter.outputs.swift == 'true' }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -42,9 +49,45 @@ jobs: swift: - 'apps/ios/**' - '.github/workflows/codeql.yml' + rust: + - '**/*.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' + - '.github/workflows/codeql.yml' + go: + - '**/*.go' + - '**/go.mod' + - '**/go.sum' + - '.github/workflows/codeql.yml' + python: + - '**/*.py' + - '.github/workflows/codeql.yml' + actions: + - '.github/**' + + - name: Pick languages to analyze + id: matrix + env: + EVENT: ${{ github.event_name }} + CHANGED: ${{ steps.filter.outputs.changes }} + run: | + if [[ "$EVENT" != pull_request ]]; then + CHANGED='["actions","go","python","rust"]' + fi + jq -cn --argjson changed "$CHANGED" '{include: ( + [{language: "javascript-typescript", "build-mode": "none"}] + + [ + {language: "actions", "build-mode": "none"}, + {language: "go", "build-mode": "autobuild"}, + {language: "python", "build-mode": "none"}, + {language: "rust", "build-mode": "none"} + | select(.language | IN($changed[])) + ] + )}' | sed 's/^/matrix=/' >> "$GITHUB_OUTPUT" analyze: name: Analyze (${{ matrix.language }}) + needs: changes runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -53,18 +96,7 @@ jobs: actions: read strategy: fail-fast: false - matrix: - include: - - language: actions - build-mode: none - - language: go - build-mode: autobuild - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none - - language: rust - build-mode: none + matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 diff --git a/docs/ci-performance.md b/docs/ci-performance.md index c7e90527a3..211cb47c0e 100644 --- a/docs/ci-performance.md +++ b/docs/ci-performance.md @@ -10,13 +10,16 @@ when services moved out of API, leaving the longest suite unsharded. root workspace patterns and tracked manifests. It sizes Vitest shards against a 140s estimated test budget and packs small suites into lanes with the same budget. Per-file rates for the large suites are measured from CI run -35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get a -conservative 1s/file, plus a per-suite startup cost. File counts deliberately +35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get +0.6s/file, plus a per-suite startup cost. That default is fitted to the slowest +cold runs of the small-suite lanes: at the old 1s/file the planner put those +lanes at ~138s when they never exceeded 86s, and spent two extra jobs on lanes +that did 2s and 7s of work. File counts deliberately include all tracked test/spec files: this can overestimate a suite, but never filters the tests actually executed. Each lane invokes the original package scripts through Turbo; Vitest owns discovery and shard assignment. -The current inventory produces 9 test lanes, including two backend shards. +The current inventory produces 10 test lanes, including three backend shards. It used to produce 21 at a 100s budget: each job pays ~30s of setup and the org runs ~16 jobs at once, so the short lanes queued the run in waves (the last test lane of run 35725366619 started 234s in). Fewer, fuller lanes cut both the @@ -44,8 +47,10 @@ cannot predict. Investigate a budget failure using the Vitest file timings: fix unnecessary real waits/repeated setup, split an oversized file, or update the measured per-file estimate. Do not simply increase the runtime budget. -Bun-native CLI and otel-helper tests share unsharded lanes with each other, never -with Vitest flags, and keep their original arguments; Rust retains its dedicated workflow. +Bun-native suites pack into the small lanes alongside Vitest suites, listed in the +lane's `bun-filters`. `run-tests.py` runs them in a second Turbo invocation without +the Vitest flags, which `bun test` rejects, and keeps their original arguments; Rust +retains its dedicated workflow. The runtime budget still applies to the Bun lanes. If those suites grow beyond it, they need runner-specific splitting rather than Vitest flags.