ci: cut the per-push job fan-out that queues CI behind the runner limit

.github/scripts/plan-tests.py

@@ -9,7 +9,9 @@
 
 ROOT = Path(__file__).resolve().parents[2]
 # Vitest seconds/file at two workers, measured on CI run 35725366619 (Duration
-# lines, excluding dependency builds). Unknown suites get the conservative default.
+# lines, excluding dependency builds). Unknown suites get the default, fitted to the
+# slowest cold small-suite lanes over a week of CI (2026-10): small-4, 105 files
+# across 8 suites, never ran longer than 86s; the old 1s/file put it at 137s.
 SECONDS_PER_FILE = {
     "@maple/api": 2.3,
     "@maple/backend": 1.6,
@@ -19,7 +21,7 @@
     "@maple/ui": 0.45,
     "@maple/domain": 0.4,
 }
-DEFAULT_SECONDS_PER_FILE = 1
+DEFAULT_SECONDS_PER_FILE = 0.6
 # Every job pays ~30s of checkout/mise/install plus Turbo dependency builds before
 # a test runs, and the org runs ~16 jobs at once, so the matrix queued in waves.
 # Fewer, fuller lanes beat many short ones on both wall clock and runner minutes.
@@ -72,7 +74,7 @@ def discover(root=ROOT):
 
 def pack(suites, prefix, args):
     groups = []
-    for name, weight in sorted(suites, key=lambda item: (-item[1], item[0])):
+    for name, weight, vitest in sorted(suites, key=lambda item: (-item[1], item[0])):
         group = next(
             (g for g in groups if g["estimated-seconds"] + weight <= TARGET_SECONDS),
             None,
@@ -82,42 +84,46 @@ def pack(suites, prefix, args):
                 "name": f"{prefix}-{len(groups) + 1}",
                 "filters": [],
                 "args": list(args),
+                "bun-filters": [],
                 "estimated-seconds": 0,
             }
             groups.append(group)
         group["filters"].append(name)
+        if not vitest:
+            group["bun-filters"].append(name)
         group["estimated-seconds"] += weight
     return groups
 
 
 def plan(suites):
-    lanes, small, bun = [], [], []
+    lanes, small = [], []
     for suite in sorted(suites, key=lambda s: s["name"]):
         name = suite["name"]
         weight = max(1, suite["files"]) * SECONDS_PER_FILE.get(
             name, DEFAULT_SECONDS_PER_FILE
         )
         if not suite["vitest"]:
-            # Bun suites share lanes with each other, never with Vitest flags.
-            bun.append((name, weight + STARTUP_SECONDS))
+            # Bun suites pack with the small Vitest suites; run-tests.py gives them
+            # their own Turbo invocation so they never receive Vitest flags.
+            small.append((name, weight + STARTUP_SECONDS, False))
             continue
         shards = math.ceil(weight / (TARGET_SECONDS - STARTUP_SECONDS))
         if shards == 1:
-            small.append((name, weight + STARTUP_SECONDS))
+            small.append((name, weight + STARTUP_SECONDS, True))
         else:
             for index in range(1, shards + 1):
                 lanes.append(
                     {
                         "name": f"{name}-{index}-of-{shards}",
                         "filters": [name],
                         "args": [f"--shard={index}/{shards}", "--maxWorkers=2"],
+                        "bun-filters": [],
                         "estimated-seconds": weight / shards + STARTUP_SECONDS,
                     }
                 )
     # First-fit decreasing packs small suites without ever running two test
     # runners on the same machine at once (Turbo concurrency=1).
     lanes.extend(pack(small, "small", ["--maxWorkers=2"]))
-    lanes.extend(pack(bun, "bun", []))
     browser_workspaces = {
         suite["name"]: suite.get("browser-workspace", "") for suite in suites
     }

.github/scripts/run-tests.py

@@ -7,12 +7,24 @@
 from pathlib import Path
 
 lane = json.loads(os.environ["TEST_LANE"])
-command = ["bun", "turbo", "test", "--concurrency=1"]
-command.extend(f"--filter={name}" for name in lane["filters"])
-if lane["args"]:
-    command.extend(["--", *lane["args"]])
+bun = lane.get("bun-filters", [])
+vitest = [name for name in lane["filters"] if name not in bun]
+base = ["bun", "turbo", "test", "--concurrency=1"]
+commands = []
+if vitest:
+    command = base + [f"--filter={name}" for name in vitest]
+    if lane["args"]:
+        command.extend(["--", *lane["args"]])
+    commands.append(command)
+# Bun suites share the lane but not its Vitest flags, which `bun test` rejects.
+if bun:
+    commands.append(base + [f"--filter={name}" for name in bun])
 start = time.monotonic()
-result = subprocess.run(command, check=False)
+# Both invocations run even after one fails, so a red lane reports every failure.
+returncode = 0
+for command in commands:
+    result = subprocess.run(command, check=False)
+    returncode = returncode or result.returncode
 elapsed = time.monotonic() - start
 # Includes cold dependency builds. The planner targets 100s, leaving ample room
 # for setup and runner variance. A single slow file must be fixed/split, not
@@ -29,4 +41,4 @@
     print(
         "::error::Test lane exceeded 240s. Inspect slow files and update the CI sizing estimates; see docs/ci-performance.md."
     )
-raise SystemExit(result.returncode or int(elapsed > budget))
+raise SystemExit(returncode or int(elapsed > budget))

.github/scripts/test_plan_tests.py

@@ -40,7 +40,7 @@ def assert_coverage(self, suites, matrix):
                     )
                 )
             else:
-                self.assertEqual(lanes[0]["args"], [])
+                self.assertIn(suite["name"], lanes[0]["bun-filters"])
         self.assertEqual(
             {s["name"] for s in suites},
             {name for lane in matrix["include"] for name in lane["filters"]},
@@ -68,17 +68,17 @@ def test_new_workspace_and_order_independence(self):
         self.assert_coverage(suites, planner.plan(suites))
         self.assertEqual(planner.plan(suites), planner.plan(list(reversed(suites))))
 
-    def test_bun_suites_share_a_lane_without_vitest_flags(self):
+    def test_bun_suites_share_small_lanes_but_are_marked(self):
         suites = [
             {"name": "bun-a", "files": 10, "vitest": False},
             {"name": "bun-b", "files": 1, "vitest": False},
             {"name": "vitest-a", "files": 1, "vitest": True},
         ]
         lanes = planner.plan(suites)["include"]
         self.assert_coverage(suites, {"include": lanes})
-        bun = [lane for lane in lanes if "bun-a" in lane["filters"]]
-        self.assertEqual(bun[0]["filters"], ["bun-a", "bun-b"])
-        self.assertEqual(bun[0]["args"], [])
+        self.assertEqual(len(lanes), 1)
+        self.assertEqual(sorted(lanes[0]["filters"]), ["bun-a", "bun-b", "vitest-a"])
+        self.assertEqual(sorted(lanes[0]["bun-filters"]), ["bun-a", "bun-b"])
 
     def test_matrix_limit_is_explicit(self):
         with self.assertRaisesRegex(ValueError, "256"):
@@ -111,7 +111,7 @@ def test_browser_install_is_selected_for_every_affected_lane(self):
 
 
 class TestRunner(unittest.TestCase):
-    def run_lane(self, args, returncode=0, elapsed=10):
+    def run_lane(self, args, returncode=0, elapsed=10, bun_filters=()):
         import json
         import os
         import runpy
@@ -122,6 +122,7 @@ def run_lane(self, args, returncode=0, elapsed=10):
             "name": "fixture",
             "filters": ["@fixture/one", "@fixture/two"],
             "args": args,
+            "bun-filters": list(bun_filters),
         }
         with (
             patch.dict(os.environ, {"TEST_LANE": json.dumps(lane)}, clear=True),
@@ -135,14 +136,14 @@ def run_lane(self, args, returncode=0, elapsed=10):
             runpy.run_path(
                 str(Path(__file__).with_name("run-tests.py")), run_name="__main__"
             )
-        return result.exception.code, run.call_args.args[0]
+        return result.exception.code, [call.args[0] for call in run.call_args_list]
 
     def test_vitest_arguments_and_failure_propagation(self):
-        code, command = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7)
+        code, commands = self.run_lane(["--shard=2/5", "--maxWorkers=2"], returncode=7)
         self.assertEqual(code, 7)
         self.assertEqual(
-            command,
-            [
+            commands,
+            [[
                 "bun",
                 "turbo",
                 "test",
@@ -152,13 +153,29 @@ def test_vitest_arguments_and_failure_propagation(self):
                 "--",
                 "--shard=2/5",
                 "--maxWorkers=2",
-            ],
+            ]],
         )
 
-    def test_bun_receives_no_vitest_arguments(self):
-        code, command = self.run_lane([])
-        self.assertEqual(code, 0)
-        self.assertNotIn("--", command)
+    def test_bun_suites_in_a_mixed_lane_get_no_vitest_arguments(self):
+        code, commands = self.run_lane(
+            ["--maxWorkers=2"], returncode=3, bun_filters=["@fixture/two"]
+        )
+        self.assertEqual(code, 3)
+        self.assertEqual(
+            commands,
+            [
+                [
+                    "bun",
+                    "turbo",
+                    "test",
+                    "--concurrency=1",
+                    "--filter=@fixture/one",
+                    "--",
+                    "--maxWorkers=2",
+                ],
+                ["bun", "turbo", "test", "--concurrency=1", "--filter=@fixture/two"],
+            ],
+        )
 
     def test_slow_passing_suite_fails_budget(self):
         code, _ = self.run_lane([], elapsed=241)

.github/workflows/codeql.yml

@@ -3,8 +3,14 @@ name: CodeQL
 # Replaces GitHub's default code-scanning setup, which cannot be path-filtered:
 # it ran the Swift analysis on a macOS runner (~10x Linux billing, 20-30 min)
 # for every PR, including the large majority that never touch apps/ios. Swift
-# now runs only when Swift could have changed, plus on the weekly schedule; the
-# other languages are cheap and keep running on every PR.
+# now runs only when Swift could have changed, plus on the weekly schedule.
+#
+# The same goes for Rust (~6 min, 22 files in apps/ingest), Go, Python and
+# Actions on pull requests: they start alongside CI and took four of the org's
+# runner slots on every push for code most PRs never touch. A PR analyzes them
+# only when their paths change; main pushes and the schedule analyze them all,
+# so the base-branch results PRs compare against stay current. JS/TS runs on
+# every event.
 #
 # Categories match default setup's (`/language:<lang>`), so alert history
 # carries over. Default setup must stay disabled (repo Settings → Code security)
@@ -32,6 +38,7 @@ jobs:
         timeout-minutes: 5
         outputs:
             swift: ${{ steps.filter.outputs.swift == 'true' }}
+            matrix: ${{ steps.matrix.outputs.matrix }}
         steps:
             - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
 
@@ -42,9 +49,45 @@ jobs:
                       swift:
                         - 'apps/ios/**'
                         - '.github/workflows/codeql.yml'
+                      rust:
+                        - '**/*.rs'
+                        - '**/Cargo.toml'
+                        - '**/Cargo.lock'
+                        - '.github/workflows/codeql.yml'
+                      go:
+                        - '**/*.go'
+                        - '**/go.mod'
+                        - '**/go.sum'
+                        - '.github/workflows/codeql.yml'
+                      python:
+                        - '**/*.py'
+                        - '.github/workflows/codeql.yml'
+                      actions:
+                        - '.github/**'
+
+            - name: Pick languages to analyze
+              id: matrix
+              env:
+                  EVENT: ${{ github.event_name }}
+                  CHANGED: ${{ steps.filter.outputs.changes }}
+              run: |
+                  if [[ "$EVENT" != pull_request ]]; then
+                    CHANGED='["actions","go","python","rust"]'
+                  fi
+                  jq -cn --argjson changed "$CHANGED" '{include: (
+                    [{language: "javascript-typescript", "build-mode": "none"}]
+                    + [
+                      {language: "actions", "build-mode": "none"},
+                      {language: "go", "build-mode": "autobuild"},
+                      {language: "python", "build-mode": "none"},
+                      {language: "rust", "build-mode": "none"}
+                      | select(.language | IN($changed[]))
+                    ]
+                  )}' | sed 's/^/matrix=/' >> "$GITHUB_OUTPUT"
 
     analyze:
         name: Analyze (${{ matrix.language }})
+        needs: changes
         runs-on: ubuntu-latest
         timeout-minutes: 30
         permissions:
@@ -53,18 +96,7 @@ jobs:
             actions: read
         strategy:
             fail-fast: false
-            matrix:
-                include:
-                    - language: actions
-                      build-mode: none
-                    - language: go
-                      build-mode: autobuild
-                    - language: javascript-typescript
-                      build-mode: none
-                    - language: python
-                      build-mode: none
-                    - language: rust
-                      build-mode: none
+            matrix: ${{ fromJSON(needs.changes.outputs.matrix) }}
         steps:
             - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6
 

docs/ci-performance.md

@@ -10,13 +10,16 @@ when services moved out of API, leaving the longest suite unsharded.
 root workspace patterns and tracked manifests. It sizes Vitest shards against
 a 140s estimated test budget and packs small suites into lanes with the same
 budget. Per-file rates for the large suites are measured from CI run
-35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get a
-conservative 1s/file, plus a per-suite startup cost. File counts deliberately
+35725366619 (backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get
+0.6s/file, plus a per-suite startup cost. That default is fitted to the slowest
+cold runs of the small-suite lanes: at the old 1s/file the planner put those
+lanes at ~138s when they never exceeded 86s, and spent two extra jobs on lanes
+that did 2s and 7s of work. File counts deliberately
 include all tracked test/spec files: this can overestimate a suite, but never
 filters the tests actually executed. Each lane invokes the original package
 scripts through Turbo; Vitest owns discovery and shard assignment.
 
-The current inventory produces 9 test lanes, including two backend shards.
+The current inventory produces 10 test lanes, including three backend shards.
 It used to produce 21 at a 100s budget: each job pays ~30s of setup and the
 org runs ~16 jobs at once, so the short lanes queued the run in waves (the last
 test lane of run 35725366619 started 234s in). Fewer, fuller lanes cut both the
@@ -44,8 +47,10 @@ cannot predict. Investigate a budget failure using the Vitest file timings:
 fix unnecessary real waits/repeated setup, split an oversized file, or update
 the measured per-file estimate. Do not simply increase the runtime budget.
 
-Bun-native CLI and otel-helper tests share unsharded lanes with each other, never
-with Vitest flags, and keep their original arguments; Rust retains its dedicated workflow.
+Bun-native suites pack into the small lanes alongside Vitest suites, listed in the
+lane's `bun-filters`. `run-tests.py` runs them in a second Turbo invocation without
+the Vitest flags, which `bun test` rejects, and keeps their original arguments; Rust
+retains its dedicated workflow.
 The runtime budget still applies to the Bun lanes. If those suites grow beyond it,
 they need runner-specific splitting rather than Vitest flags.