diff --git a/.github/scripts/plan-tests.py b/.github/scripts/plan-tests.py index 67237490fe..89b8c97345 100644 --- a/.github/scripts/plan-tests.py +++ b/.github/scripts/plan-tests.py @@ -9,7 +9,9 @@ ROOT = Path(__file__).resolve().parents[2] # Vitest seconds/file at two workers, measured on CI run 35725366619 (Duration -# lines, excluding dependency builds). Unknown suites get the conservative default. +# lines, excluding dependency builds). Unknown suites get the default, fitted to the +# slowest cold small-suite lanes over a week of CI (2026-10): small-4, 105 files +# across 8 suites, never ran longer than 86s; the old 1s/file put it at 137s. SECONDS_PER_FILE = { "@maple/api": 2.3, "@maple/backend": 1.6, @@ -19,7 +21,7 @@ "@maple/ui": 0.45, "@maple/domain": 0.4, } -DEFAULT_SECONDS_PER_FILE = 1 +DEFAULT_SECONDS_PER_FILE = 0.6 # Every job pays ~30s of checkout/mise/install plus Turbo dependency builds before # a test runs, and the org runs ~16 jobs at once, so the matrix queued in waves. # Fewer, fuller lanes beat many short ones on both wall clock and runner minutes. @@ -72,7 +74,7 @@ def discover(root=ROOT): def pack(suites, prefix, args): groups = [] - for name, weight in sorted(suites, key=lambda item: (-item[1], item[0])): + for name, weight, vitest in sorted(suites, key=lambda item: (-item[1], item[0])): group = next( (g for g in groups if g["estimated-seconds"] + weight <= TARGET_SECONDS), None, @@ -82,28 +84,32 @@ def pack(suites, prefix, args): "name": f"{prefix}-{len(groups) + 1}", "filters": [], "args": list(args), + "bun-filters": [], "estimated-seconds": 0, } groups.append(group) group["filters"].append(name) + if not vitest: + group["bun-filters"].append(name) group["estimated-seconds"] += weight return groups def plan(suites): - lanes, small, bun = [], [], [] + lanes, small = [], [] for suite in sorted(suites, key=lambda s: s["name"]): name = suite["name"] weight = max(1, suite["files"]) * SECONDS_PER_FILE.get( name, DEFAULT_SECONDS_PER_FILE ) if not suite["vitest"]: - # Bun suites share lanes with each other, never with Vitest flags. - bun.append((name, weight + STARTUP_SECONDS)) + # Bun suites pack with the small Vitest suites; run-tests.py gives them + # their own Turbo invocation so they never receive Vitest flags. + small.append((name, weight + STARTUP_SECONDS, False)) continue shards = math.ceil(weight / (TARGET_SECONDS - STARTUP_SECONDS)) if shards == 1: - small.append((name, weight + STARTUP_SECONDS)) + small.append((name, weight + STARTUP_SECONDS, True)) else: for index in range(1, shards + 1): lanes.append( @@ -111,13 +117,13 @@ def plan(suites): "name": f"{name}-{index}-of-{shards}", "filters": [name], "args": [f"--shard={index}/{shards}", "--maxWorkers=2"], + "bun-filters": [], "estimated-seconds": weight / shards + STARTUP_SECONDS, } ) # First-fit decreasing packs small suites without ever running two test # runners on the same machine at once (Turbo concurrency=1). lanes.extend(pack(small, "small", ["--maxWorkers=2"])) - lanes.extend(pack(bun, "bun", [])) browser_workspaces = { suite["name"]: suite.get("browser-workspace", "") for suite in suites } diff --git a/.github/scripts/run-tests.py b/.github/scripts/run-tests.py index 44c3547933..e552b23b20 100644 --- a/.github/scripts/run-tests.py +++ b/.github/scripts/run-tests.py @@ -7,12 +7,24 @@ from pathlib import Path lane = json.loads(os.environ["TEST_LANE"]) -command = ["bun", "turbo", "test", "--concurrency=1"] -command.extend(f"--filter={name}" for name in lane["filters"]) -if lane["args"]: - command.extend(["--", *lane["args"]]) +bun = lane.get("bun-filters", []) +vitest = [name for name in lane["filters"] if name not in bun] +base = ["bun", "turbo", "test", "--concurrency=1"] +commands = [] +if vitest: + command = base + [f"--filter={name}" for name in vitest] + if lane["args"]: + command.extend(["--", *lane["args"]]) + commands.append(command) +# Bun suites share the lane but not its Vitest flags, which `bun test` rejects. +if bun: + commands.append(base + [f"--filter={name}" for name in bun]) start = time.monotonic() -result = subprocess.run(command, check=False) +# Both invocations run even after one fails, so a red lane reports every failure. +returncode = 0 +for command in commands: + result = subprocess.run(command, check=False) + returncode = returncode or result.returncode elapsed = time.monotonic() - start # Includes cold dependency builds. The planner targets 100s, leaving ample room # for setup and runner variance. A single slow file must be fixed/split, not @@ -29,4 +41,4 @@ print( "::error::Test lane exceeded 240s. Inspect slow files and update the CI sizing estimates; see docs/ci-performance.md." ) -raise SystemExit(result.returncode or int(elapsed > budget)) +raise SystemExit(returncode or int(elapsed > budget)) diff --git a/.github/scripts/test_plan_tests.py b/.github/scripts/test_plan_tests.py index 4c04c1f205..21747b0cdc 100644 --- a/.github/scripts/test_plan_tests.py +++ b/.github/scripts/test_plan_tests.py @@ -40,7 +40,7 @@ def assert_coverage(self, suites, matrix): ) ) else: - self.assertEqual(lanes[0]["args"], []) + self.assertIn(suite["name"], lanes[0]["bun-filters"]) self.assertEqual( {s["name"] for s in suites}, {name for lane in matrix["include"] for name in lane["filters"]}, @@ -68,7 +68,7 @@ def test_new_workspace_and_order_independence(self): self.assert_coverage(suites, planner.plan(suites)) self.assertEqual(planner.plan(suites), planner.plan(list(reversed(suites)))) - def test_bun_suites_share_a_lane_without_vitest_flags(self): + def test_bun_suites_share_small_lanes_but_are_marked(self): suites = [ {"name": "bun-a", "files": 10, "vitest": False}, {"name": "bun-b", "files": 1, "vitest": False}, @@ -76,9 +76,9 @@ def test_bun_suites_share_a_lane_without_vitest_flags(self): ] lanes = planner.plan(suites)["include"] self.assert_coverage(suites, {"include": lanes}) - bun = [lane for lane in lanes if "bun-a" in lane["filters"]] - self.assertEqual(bun[0]["filters"], ["bun-a", "bun-b"]) - self.assertEqual(bun[0]["args"], []) + self.assertEqual(len(lanes), 1) + self.assertEqual(sorted(lanes[0]["filters"]), ["bun-a", "bun-b", "vitest-a"]) + self.assertEqual(sorted(lanes[0]["bun-filters"]), ["bun-a", "bun-b"]) def test_matrix_limit_is_explicit(self): with self.assertRaisesRegex(ValueError, "256"): @@ -111,7 +111,7 @@ def test_browser_install_is_selected_for_every_affected_lane(self): class TestRunner(unittest.TestCase): - def run_lane(self, args, returncode=0, elapsed=10): + def run_lane(self, args, returncode=0, elapsed=10, bun_filters=()): import json import os import runpy @@ -122,6 +122,7 @@ def run_lane(self, args, returncode=0, elapsed=10): "name": "fixture", "filters": ["@fixture/one", "@fixture/two"], "args": args, + "bun-filters": list(bun_filters), } with ( patch.dict(os.environ, {"TEST_LANE": json.dumps(lane)}, clear=True), @@ -135,14 +136,14 @@ def run_lane(self, args, returncode=0, elapsed=10): runpy.run_path( str(Path(__file__).with_name("run-tests.py")), run_name="__main__" ) - return result.exception.code, run.call_args.args[0] + return result.exception.code, [call.args[0] for call in run.call_args_list] def test_vitest_arguments_and_failure_propagation(self): - code, command = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7) + code, commands = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7) self.assertEqual(code, 7) self.assertEqual( - command, - [ + commands, + [[ "bun", "turbo", "test", @@ -152,13 +153,29 @@ def test_vitest_arguments_and_failure_propagation(self): "--", "--shard=2/5", "--maxWorkers=2", - ], + ]], ) - def test_bun_receives_no_vitest_arguments(self): - code, command = self.run_lane([]) - self.assertEqual(code, 0) - self.assertNotIn("--", command) + def test_bun_suites_in_a_mixed_lane_get_no_vitest_arguments(self): + code, commands = self.run_lane( + ["--maxWorkers=2"], returncode=3, bun_filters=["@fixture/two"] + ) + self.assertEqual(code, 3) + self.assertEqual( + commands, + [ + [ + "bun", + "turbo", + "test", + "--concurrency=1", + "--filter=@fixture/one", + "--", + "--maxWorkers=2", + ], + ["bun", "turbo", "test", "--concurrency=1", "--filter=@fixture/two"], + ], + ) def test_slow_passing_suite_fails_budget(self): code, _ = self.run_lane([], elapsed=241) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index cbbfeac3a0..311ce769fc 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -3,8 +3,14 @@ name: CodeQL # Replaces GitHub's default code-scanning setup, which cannot be path-filtered: # it ran the Swift analysis on a macOS runner (~10x Linux billing, 20-30 min) # for every PR, including the large majority that never touch apps/ios. Swift -# now runs only when Swift could have changed, plus on the weekly schedule; the -# other languages are cheap and keep running on every PR. +# now runs only when Swift could have changed, plus on the weekly schedule. +# +# The same goes for Rust (~6 min, 22 files in apps/ingest), Go, Python and +# Actions on pull requests: they start alongside CI and took four of the org's +# runner slots on every push for code most PRs never touch. A PR analyzes them +# only when their paths change; main pushes and the schedule analyze them all, +# so the base-branch results PRs compare against stay current. JS/TS runs on +# every event. # # Categories match default setup's (`/language:`), so alert history # carries over. Default setup must stay disabled (repo Settings → Code security) @@ -32,6 +38,7 @@ jobs: timeout-minutes: 5 outputs: swift: ${{ steps.filter.outputs.swift == 'true' }} + matrix: ${{ steps.matrix.outputs.matrix }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -42,9 +49,45 @@ jobs: swift: - 'apps/ios/**' - '.github/workflows/codeql.yml' + rust: + - '**/*.rs' + - '**/Cargo.toml' + - '**/Cargo.lock' + - '.github/workflows/codeql.yml' + go: + - '**/*.go' + - '**/go.mod' + - '**/go.sum' + - '.github/workflows/codeql.yml' + python: + - '**/*.py' + - '.github/workflows/codeql.yml' + actions: + - '.github/**' + + - name: Pick languages to analyze + id: matrix + env: + EVENT: ${{ github.event_name }} + CHANGED: ${{ steps.filter.outputs.changes }} + run: | + if [[ "$EVENT" != pull_request ]]; then + CHANGED='["actions","go","python","rust"]' + fi + jq -cn --argjson changed "$CHANGED" '{include: ( + [{language: "javascript-typescript", "build-mode": "none"}] + + [ + {language: "actions", "build-mode": "none"}, + {language: "go", "build-mode": "autobuild"}, + {language: "python", "build-mode": "none"}, + {language: "rust", "build-mode": "none"} + | select(.language | IN($changed[])) + ] + )}' | sed 's/^/matrix=/' >> "$GITHUB_OUTPUT" analyze: name: Analyze (${{ matrix.language }}) + needs: changes runs-on: ubuntu-latest timeout-minutes: 30 permissions: @@ -53,18 +96,7 @@ jobs: actions: read strategy: fail-fast: false - matrix: - include: - - language: actions - build-mode: none - - language: go - build-mode: autobuild - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none - - language: rust - build-mode: none + matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 diff --git a/docs/ci-performance.md b/docs/ci-performance.md index c7e90527a3..211cb47c0e 100644 --- a/docs/ci-performance.md +++ b/docs/ci-performance.md @@ -10,13 +10,16 @@ when services moved out of API, leaving the longest suite unsharded. root workspace patterns and tracked manifests. It sizes Vitest shards against a 140s estimated test budget and packs small suites into lanes with the same budget. Per-file rates for the large suites are measured from CI run -35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get a -conservative 1s/file, plus a per-suite startup cost. File counts deliberately +35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get +0.6s/file, plus a per-suite startup cost. That default is fitted to the slowest +cold runs of the small-suite lanes: at the old 1s/file the planner put those +lanes at ~138s when they never exceeded 86s, and spent two extra jobs on lanes +that did 2s and 7s of work. File counts deliberately include all tracked test/spec files: this can overestimate a suite, but never filters the tests actually executed. Each lane invokes the original package scripts through Turbo; Vitest owns discovery and shard assignment. -The current inventory produces 9 test lanes, including two backend shards. +The current inventory produces 10 test lanes, including three backend shards. It used to produce 21 at a 100s budget: each job pays ~30s of setup and the org runs ~16 jobs at once, so the short lanes queued the run in waves (the last test lane of run 35725366619 started 234s in). Fewer, fuller lanes cut both the @@ -44,8 +47,10 @@ cannot predict. Investigate a budget failure using the Vitest file timings: fix unnecessary real waits/repeated setup, split an oversized file, or update the measured per-file estimate. Do not simply increase the runtime budget. -Bun-native CLI and otel-helper tests share unsharded lanes with each other, never -with Vitest flags, and keep their original arguments; Rust retains its dedicated workflow. +Bun-native suites pack into the small lanes alongside Vitest suites, listed in the +lane's `bun-filters`. `run-tests.py` runs them in a second Turbo invocation without +the Vitest flags, which `bun test` rejects, and keeps their original arguments; Rust +retains its dedicated workflow. The runtime budget still applies to the Bun lanes. If those suites grow beyond it, they need runner-specific splitting rather than Vitest flags.