feat(tooling): tracked flaky-test quarantine (cli, vitest-wired)

.github/workflows/test.yml

@@ -24,7 +24,27 @@ jobs:
             # inspect alone does not pull; silent skip was fake B1 coverage (#1911).
             - run: docker pull mcr.microsoft.com/powershell:latest
             - run: bun run test:e2e -- terminal-wrap-fidelity.spec.ts composer-copy.spec.ts session-list-scroll.spec.ts cold-initial-tail.spec.ts scrollbar-auto-hide.spec.ts
-            - run: bun run test
+            # Fork: cli tests run through the flaky-quarantine gate (same
+            # script the local remat gate uses — config/flaky-quarantine.yaml
+            # is fork-only data, see docs/tooling/flaky-quarantine.md).
+            # Upstream/other forks: unchanged plain `bun run test`.
+            - name: Run tests
+              env:
+                  # flaky-quarantine.mjs validate/check shell out to `gh issue
+                  # view` to confirm a quarantine entry's issue is still open
+                  # (requirement: a closed-issue entry must fail loudly, not
+                  # sit there forever). Without this, every gh call returns
+                  # UNKNOWN on a fresh runner and the gate fails for the wrong
+                  # reason — matches the GH_TOKEN/GITHUB_TOKEN pattern already
+                  # used by codex-mention-response.yml / issue-auto-response.yml.
+                  GH_TOKEN: ${{ github.token }}
+                  GITHUB_TOKEN: ${{ github.token }}
+              run: |
+                  if [[ "${{ github.repository_owner }}" == "heavygee" ]]; then
+                      bash scripts/tooling/flaky-quarantine-gate.sh "$PWD"
+                  else
+                      bun run test
+                  fi
             # Fork has no windows-latest budget; cover the same file on Linux.
             - name: Codex MCP proxy unit (Linux stand-in for windows-codex-mcp)
               if: github.repository_owner == 'heavygee'

config/flaky-quarantine.yaml

@@ -0,0 +1,128 @@
+# Flaky-test quarantine registry — tracked, not ignored.
+#
+# FORK-ONLY DATA: these entries are fork-specific CI observations and issue
+# links (heavygee/hapi). The mechanism that reads this file
+# (scripts/tooling/flaky-quarantine.mjs + lib/flaky-quarantine-core.mjs) is
+# generic — any repo with CI would want it — and is a candidate for an
+# eventual standalone/upstream extraction. Keep the split: this file carries
+# the fork's observations, not the other way around.
+#
+# WHY THIS EXISTS (2026-10-15): operator-directed after two unrelated pieces
+# of work were held hostage by red lights nobody could fix locally — PR #222
+# (iOS app-build failed 3 different ways on a diff that never touches the
+# failing screens, heavygee/hapi#273) and the #270 idle-gate remat (tripped
+# by agentCliGuard.test.ts under full-suite timing, heavygee/hapi#274). Each
+# time, an agent escalated to the operator for a judgement call on a test it
+# didn't write. "Red doesn't really mean red" is the disease this fixes.
+#
+# Evidence bar — do not add an entry without ONE of:
+#   - "isolation passes, full-suite fails" (both runs actually observed, not
+#     asserted — log what you saw in the linked issue)
+#   - ">=2 failures with DIFFERENT assertions/signatures on the same
+#     unchanged commit" (same bug retried is not flake; this is a regression)
+# "Failed once" is never sufficient evidence.
+#
+# Checked by: scripts/tooling/flaky-quarantine.mjs (validate|check), called
+# from both CI (.github/workflows/test.yml, ios.yml) and the local remat gate
+# (hapi-driver-rebuild.sh) via scripts/tooling/flaky-quarantine-gate.sh — ONE
+# script, so the two gates cannot drift onto different lists.
+#
+# Fields (all required unless noted):
+#   id             - stable test identifier.
+#                      vitest: "<package>:<ancestorTitles-space-joined> <title>"
+#                              e.g. produced verbatim by vitest's own
+#                              --reporter=json `fullName` field — see
+#                              `flaky-quarantine.mjs from-vitest-json`.
+#                      XCTest: "ios:<TestClass>/<testMethod>"
+#   owner          - person/agent handle responsible for eventually fixing or
+#                    removing this entry. No anonymous entries.
+#   issue          - heavygee/hapi issue number tracking the root cause. MUST
+#                    be OPEN (validated against the live GitHub issue state,
+#                    not just present) — a quarantined test whose issue is
+#                    closed is either fixed or abandoned, and both deserve to
+#                    fail loudly again, not sit here forever.
+#   signature      - one or more " && "-joined required substrings of the
+#                    known failure text — ALL must be present in the message
+#                    for a match. A failure under this id that doesn't match
+#                    is treated as a NEW failure and BLOCKS — this is what
+#                    stops quarantine from hiding a real regression that
+#                    happens to land in the same test. Prefer combining the
+#                    assertion text with its stack-trace call site (e.g.
+#                    "expected true to be false && myTest.ts:42:") — either
+#                    alone can over-match (generic assertion text can hit a
+#                    different assertion in the same test; a bare call site
+#                    can match a different kind of failure thrown from the
+#                    same line). See the cli: entry below for the real case
+#                    that forced this (cold review, 2026-10-15).
+#   added          - ISO date quarantined.
+#   expires        - ISO date. Default policy is +14 days from `added`. Past
+#                    this date the entry excuses nothing — it is not a grace
+#                    period, the entry simply drops out of the active set and
+#                    the test goes back to blocking until fixed, deleted, or
+#                    deliberately renewed.
+#   renewal_count  - (optional, default 0) each +1 permits 14 more days
+#                    beyond the default (enforced by the validator: expires
+#                    may not be more than 14 + 14*renewal_count days after
+#                    added). You cannot push the date out without bumping
+#                    this — the renewal itself is a diff.
+#   renewal_reason - required whenever renewal_count > 0. A concrete reason
+#                    ("renewed: still intermittent under CI load, see #274"),
+#                    not a bare date bump. Older renewal reasons live in git
+#                    log on this file, not here — this field is the latest.
+#
+# Ceiling: scripts/tooling/flaky-quarantine.mjs defaults to 8 active entries.
+# Past that, the gate fails regardless of test outcomes until the registry is
+# cleaned up — the count is a forcing function, not a decoration.
+
+quarantine:
+  - id: "cli:agentCliGuard prunes crash-stale registering/<deadPid> so list-models is not pinned"
+    owner: "heavygee"
+    issue: 274
+    # Two cold-review rounds landed here (2026-10-15):
+    # Round 1: "expected true to be false" alone is too generic — this test
+    # has a SECOND assertion two lines below (existsSync(dir)).toBe(false))
+    # that fails with byte-identical text for a real, different reason.
+    # Round 2: the stack-frame line ALONE is also not enough — if
+    # isAgentAcpTransportActive() itself started throwing instead of
+    # returning the wrong boolean, the thrown error's stack would still
+    # name this same line, and a location-only signature would wrongly
+    # excuse a completely different failure mode.
+    # `signature` supports " && "-joined required substrings — ALL must be
+    # present. Combining the assertion text with its call site closes both
+    # gaps: neither alone can satisfy this quarantine's excuse condition.
+    # Line-number signatures are brittle to lines being inserted above 309 in
+    # this file — if that happens, this stops matching and BLOCKS (fails
+    # closed), which forces someone to look rather than silently re-matching.
+    signature: "expected true to be false && agentCliGuard.test.ts:309:"
+    added: "2026-10-15"
+    expires: "2026-10-29"
+
+  # NOT WIRED YET (see docs/tooling/flaky-quarantine.md — the ios adapter is
+  # built and proven against real logs, but has no macOS runner to verify the
+  # live ios.yml integration). Known limitation carried over from the cold
+  # review that caught the cli entry's generic-signature problem above: these
+  # three ios signatures are the most specific substring captured from the
+  # real failing run WITHOUT including per-run-variable text (memory
+  # addresses, measured pixel values). They have NOT been checked against the
+  # iOS test source for a sibling assertion in the same method that could
+  # produce the same generic text — do that check before wiring these live.
+  - id: "ios:ToolTranscriptPresentationTests/testPlanPublicationUpdatesRecyclingAndInspectorPreserveTheDocumentAndAnchor"
+    owner: "heavygee"
+    issue: 273
+    signature: "XCTAssertNil failed: \"<UICollectionViewCell"
+    added: "2026-10-15"
+    expires: "2026-10-29"
+
+  - id: "ios:HomeFilterPresentationTests/testFilteringResetsScrollButLiveUpdatesAndChatReturnPreserveIt"
+    owner: "heavygee"
+    issue: 273
+    signature: "XCTUnwrap failed: expected non-nil value of type \"UICollectionView\""
+    added: "2026-10-15"
+    expires: "2026-10-29"
+
+  - id: "ios:TranscriptTypographyTests/testShrinkingTextClampsAnOffsetBeyondTheNewRowHeight"
+    owner: "heavygee"
+    issue: 273
+    signature: "XCTAssertEqualWithAccuracy failed"
+    added: "2026-10-15"
+    expires: "2026-10-29"

docs/tooling/flaky-quarantine.md

@@ -0,0 +1,101 @@
+# Flaky-test quarantine
+
+## Why this exists
+
+Two unrelated pieces of work were held hostage by red lights nobody could fix
+locally (2026-10-15): [PR #222](https://github.com/heavygee/hapi/pull/222)
+blocked on iOS `app-build` failing 3 different ways on a diff that never
+touches the failing screens ([#273](https://github.com/heavygee/hapi/issues/273)),
+and the #270 idle-gate remat tripped by `agentCliGuard.test.ts` under
+full-suite timing ([#274](https://github.com/heavygee/hapi/issues/274)). Each
+time, an agent escalated to the operator for a judgement call on a test it
+didn't write. A tolerated red light teaches everyone that red doesn't really
+mean red — this is the fix.
+
+**A quarantined test still runs and still reports.** This is a tracked
+quarantine, not an ignore list: every entry has an owner, an open issue, a
+captured failure signature, and an expiry. Nothing here hides a test from CI
+output; it only decides whether a *known* failure may be excused for
+gate-pass purposes.
+
+## The three rules
+
+1. **Known id + matching failure signature → excused (non-blocking).**
+2. **Known id + a DIFFERENT signature → BLOCKS.** Id-matching alone can't
+   tell "the same old flake" from "a new bug that landed in the same test" —
+   comparing the failure text closes most of that gap. A warning nobody must
+   act on is exactly the mechanism that trained everyone to ignore red in the
+   first place, so a mismatch blocks rather than warns.
+3. **An expired entry excuses nothing.** It is not a grace period — the
+   entry drops out of the active set, and the next failure under that id is
+   an ordinary blocking failure until someone renews (with a reason) or
+   removes it.
+
+Plus a **ceiling** (default 8) on the number of active entries: past it, the
+gate fails regardless of whether today's tests pass, until the registry is
+cleaned up. The count is a forcing function, not a decoration.
+
+## Where the data lives
+
+`config/flaky-quarantine.yaml` — one file, read by both consumers (the local
+remat gate and CI), so they cannot drift onto different lists. Field
+reference and evidence bar are documented in the file's own header comment.
+
+**Fork-only data; generic mechanism.** The YAML's content (which tests, which
+issues, which owners) is specific to `heavygee/hapi`'s own CI observations.
+The code that reads it (`scripts/tooling/flaky-quarantine.mjs` +
+`scripts/tooling/lib/flaky-quarantine-core.mjs`) is generic — any repo with
+CI would want it — and is a candidate for an eventual standalone/upstream
+extraction. Don't let fork-specific assumptions leak into the script.
+
+## How it's wired
+
+```
+                      config/flaky-quarantine.yaml
+                        /                        \
+   scripts/tooling/flaky-quarantine-gate.sh      ios/scripts/test-transcript.sh
+     (hapi-driver-rebuild.sh --verify,             (ios.yml — DRAFTED, not yet
+      .github/workflows/test.yml)                   applied; see below)
+```
+
+- **cli (vitest):** `flaky-quarantine-gate.sh <repo-root>` runs hub/web/
+  shared/relay exactly as before (unchanged, fully blocking), then runs
+  `cli`'s tests via `vitest run --reporter=default --reporter=json
+  --outputFile.json=...`, converts the JSON report
+  (`flaky-quarantine.mjs from-vitest-json`) and feeds it to
+  `flaky-quarantine.mjs check`. Same script, called from both
+  `hapi-driver-rebuild.sh`'s `--verify` test step and
+  `.github/workflows/test.yml`'s `test` job (fork-only — upstream and other
+  forks keep plain `bun run test`, see the `github.repository_owner` guard in
+  that file).
+- **iOS (XCTest):** `flaky-quarantine.mjs from-xcodebuild-log` parses
+  xcodebuild's `error: -[Class method] : <message>` failure lines into the
+  same normalized `{id, failed, message}` shape, proven against the three
+  real PR #222 attempt logs (`ios:<TestClass>/<testMethod>` ids). **Not yet
+  wired into `ios/scripts/test-transcript.sh` / `ios.yml`** — this host has
+  no macOS runner, so the actual shell integration (capturing xcodebuild's
+  stdout, `bun` on the GitHub-hosted `macos-15` runner, which `ios.yml`
+  doesn't currently set up at all) could not be executed or verified here.
+  Wiring it blind into a live gating workflow risked turning a flake into a
+  hard break. Apply + smoke-test on an actual macOS run before trusting it.
+
+## Validating the registry
+
+```bash
+bun scripts/tooling/flaky-quarantine.mjs validate
+```
+
+Checks schema (owner/issue/signature/dates present and sane, renewal math
+consistent with `renewal_count`), confirms every `issue` is still OPEN on
+GitHub, and reports the active count against the ceiling. Exit 2 on any
+failure — safe to run standalone in a lint step.
+
+## Negative controls
+
+`scripts/tooling/flaky-quarantine.test.sh` (picked up automatically by the
+existing `tooling-tests` CI job, no workflow change needed) proves, with
+synthetic fixtures, that the gate can actually go red three ways before
+anyone trusts it to go green: a quarantined test failing with its known
+signature does not block; the same id failing with a different signature
+blocks; an expired entry blocks even with a matching signature. Plus schema
+and ceiling checks.

scripts/tooling/flaky-quarantine-gate.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# flaky-quarantine-gate.sh <repo-root>
+#
+# Shared `bun run test` replacement for BOTH the local remat gate
+# (hapi-driver-rebuild.sh) and CI (.github/workflows/test.yml) — the SAME
+# script, so the two gates cannot drift onto different interpretations of
+# config/flaky-quarantine.yaml. See docs/tooling/flaky-quarantine.md.
+#
+# Only `cli` is quarantine-aware today — that is where the two seed entries
+# (heavygee/hapi#273's iOS siblings are a separate `ios:` id space checked by
+# ios/scripts/test-transcript.sh; #274 is the cli-side entry here) live.
+# hub/web/shared/relay run exactly as before, unchanged, fully blocking.
+# Extending quarantine awareness to another package is the same
+# vitest-json-reporter + adapter + check sequence repeated for that package.
+set -euo pipefail
+
+REPO_ROOT="${1:?usage: flaky-quarantine-gate.sh <repo-root>}"
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Honor an inherited $BUN (hapi-driver-rebuild.sh resolves "${BUN:-$HOME/.bun/bin/bun}"
+# but does not export it — callers must pass it through explicitly, e.g.
+# `BUN="$BUN" bash flaky-quarantine-gate.sh ...`). Falling back to bare `bun`
+# only works when it happens to be on PATH, which the caller's own fallback
+# does not guarantee (cold review, 2026-10-15).
+BUN="${BUN:-bun}"
+
+run_plain() {
+    echo "Running $1 tests..."
+    (cd "$REPO_ROOT/$1" && "$BUN" run test)
+}
+
+run_plain hub
+run_plain web
+run_plain shared
+run_plain relay
+
+# Whole-registry health (ceiling + schema + issue-open-ness) is its OWN gate,
+# run once, clearly attributed — NOT folded into the package-scoped `check`
+# below. Without this split, an unrelated package's entry going bad (e.g. an
+# `ios:` issue getting closed) would fail THIS cli-only run for a reason that
+# has nothing to do with cli, which is exactly the cross-package coupling the
+# --package flag on `check` exists to avoid. The ceiling itself is
+# deliberately whole-registry (not per-package) — see flaky-quarantine.mjs.
+echo "Validating quarantine registry (whole-file: ceiling + schema + issue-open)..."
+if ! "$BUN" "$HERE/flaky-quarantine.mjs" validate --registry "$REPO_ROOT/config/flaky-quarantine.yaml"; then
+    echo "ERROR: config/flaky-quarantine.yaml itself is invalid or over ceiling — fix the registry before any gate can trust it." >&2
+    exit 1
+fi
+
+echo "Running cli tests (quarantine-aware)..."
+CLI_JSON="$(mktemp -t flaky-quarantine-cli-XXXXXX.json)"
+cleanup() { rm -f "$CLI_JSON" "${NORMALIZED:-}"; }
+trap cleanup EXIT
+
+set +e
+(cd "$REPO_ROOT/cli" && "$BUN" run tools:unpack && "$BUN" x vitest run --reporter=default --reporter=json --outputFile.json="$CLI_JSON")
+VITEST_RC=$?
+set -e
+
+if [[ ! -s "$CLI_JSON" ]]; then
+    echo "ERROR: cli test run produced no JSON report (a setup step likely failed before vitest ran, exit $VITEST_RC) — hard failure, not quarantine-eligible" >&2
+    exit 1
+fi
+
+NORMALIZED="$(mktemp -t flaky-quarantine-results-XXXXXX.json)"
+"$BUN" "$HERE/flaky-quarantine.mjs" from-vitest-json "$CLI_JSON" --package cli >"$NORMALIZED"
+# --runner-exit-code: a JSON report CAN exist (vitest wrote it) while vitest
+# itself still exited non-zero with zero attributable per-test failures — a
+# collection/import error or zero tests discovered. Without passing this
+# through, that failure mode would silently normalize to an empty results
+# list and the gate would report a clean pass (cold review, 2026-10-15).
+"$BUN" "$HERE/flaky-quarantine.mjs" check --results "$NORMALIZED" --package cli --registry "$REPO_ROOT/config/flaky-quarantine.yaml" --runner-exit-code "$VITEST_RC"