From 560b60b3adcbe1e17f4726c5d308f6fc4421d061 Mon Sep 17 00:00:00 2001 From: Jared Lunde Date: Wed, 7 Oct 2026 07:54:21 -0700 Subject: [PATCH] agent: CI scans release binaries for test-seam names The authority on "no test seam ships" is now the artifact itself: CI's `build (release)` job runs `mise check:release-seams`, which greps every top-level release binary for the BEYOND_AI_AGENT_TEST_ prefix and fails on any hit. tests/release_seams.rs stays as the fast source-level warning and pins the scanner (it must fail on the debug build and on a concat!-built name). Split out of #159; its restore-window half is dropped. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JimHGjsfk2Ktm5GxyZJKKk --- .github/workflows/ci.yml | 3 ++ crates/agent/ARCHITECTURE.md | 2 +- crates/agent/scripts/scan-release-seams.sh | 26 +++++++++++++ crates/agent/tests/release_seams.rs | 45 +++++++++++++++++++++- mise.toml | 4 ++ 5 files changed, 78 insertions(+), 2 deletions(-) create mode 100755 crates/agent/scripts/scan-release-seams.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9238aac1..cee20f77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,6 +299,9 @@ jobs: - uses: jdx/mise-action@v4 - uses: Swatinem/rust-cache@v2 - run: mise build:rs:release + # The authority on "no test seam ships": the release binaries themselves, scanned for the + # seam prefix (a grep over what was just built; seconds). + - run: mise check:release-seams # Mutation testing on what this PR changed: every line the diff touches in the gateway and the # catalog is mutated, and a mutant the tests don't catch fails the job. Cost scales with the diff # (each mutant is a rebuild plus a test run), so the mutants are split across 8 runners: a typical diff --git a/crates/agent/ARCHITECTURE.md b/crates/agent/ARCHITECTURE.md index ce45661a..0c8df99a 100644 --- a/crates/agent/ARCHITECTURE.md +++ b/crates/agent/ARCHITECTURE.md @@ -4316,7 +4316,7 @@ backend an operator points it at. | `BEYOND_AI_AGENT_TEST_SLOW_SESSION_OPEN_MS` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: makes every `Persistence::open` sleep this many milliseconds, standing in for a slow network-backed session directory. `tests/serve_startup_stall.rs` uses it to prove session start-up is off the shared runtime thread. A release binary contains no latency-injection path at all. | | `AI_AGENT_OAUTH_CALLBACK_HOST` | unset (`127.0.0.1`) | The host the fixed-port OAuth login callback listener (Anthropic 53692, OpenAI Codex 1455) binds. **Loopback only** — an address in `127.0.0.0/8`, `::1`, or `localhost` — because the listener receives an authorization code; anything else (`0.0.0.0`, a routable address, a name) fails the login with a clear error before any bind (`oauth::callback_server::callback_host`). Tests give a login its own loopback address so concurrent runs do not collide on the fixed port. | | `BEYOND_AI_AGENT_TEST_SLOW_CHECKPOINT_MS` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: makes every mid-run checkpoint write in `serve` take this many milliseconds longer, inside the write itself, standing in for a slow disk — so an acknowledgement sent before the write would be caught. `tests/serve_session_lifecycle.rs` uses it to prove a tool does not start before its `tool_use` is on disk. A release binary contains no latency-injection path at all. | -| `BEYOND_AI_AGENT_TEST_FAIL_CHECKPOINT` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: fails `serve`'s checkpoint writes — `all` of them, or only the `n`th (a number), counted per session so a daemon's sessions do not shift each other's count (`serve_session_isolation`: `an_injected_checkpoint_failure_counts_each_sessions_writes`). `tests/serve_session_lifecycle.rs` uses it to prove a failed write before a tool fails closed (the tool never runs) and that a best-effort one is superseded. A release binary contains no fault-injection path and none of these names (`tests/release_seams.rs`). | +| `BEYOND_AI_AGENT_TEST_FAIL_CHECKPOINT` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: fails `serve`'s checkpoint writes — `all` of them, or only the `n`th (a number), counted per session so a daemon's sessions do not shift each other's count (`serve_session_isolation`: `an_injected_checkpoint_failure_counts_each_sessions_writes`). `tests/serve_session_lifecycle.rs` uses it to prove a failed write before a tool fails closed (the tool never runs) and that a best-effort one is superseded. A release binary contains no fault-injection path and none of these names: CI's `build (release)` job scans every release binary for the `BEYOND_AI_AGENT_TEST_` prefix (`mise check:release-seams`, `scripts/scan-release-seams.sh`) and fails on any hit; `tests/release_seams.rs` is the fast source-level warning and pins the scanner. | | `BEYOND_AI_AGENT_TEST_SLOW_SESSION_END_MS` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: delays a failed daemon session's `error` broadcast after its loop has ended. `tests/serve_service_session_end.rs` uses it to prove the reason still reaches the client before the close. A release binary contains no latency-injection path at all. | | `BEYOND_AI_AGENT_TEST_SLOW_ATTACH_MS` | unset (no delay) | **Debug builds only** (`#[cfg(debug_assertions)]`), and only for tests: delays a daemon connection's registration of its output. `tests/serve_service_session_end.rs` uses it to prove a connection registering after its session ended is still told why. A release binary contains no latency-injection path at all. | | `--idle-timeout-ms` / `AI_AGENT_IDLE_TIMEOUT_MS` (`run`-only) | `agent_core::client::READ_TIMEOUT` (~600s) | Task #19 (pi-parity feature): overrides the gateway HTTP client's idle-read timeout between response chunks (`GatewayClient::with_idle_timeout`, previously unused anywhere in this codebase) — matters most for a direct-routed/custom `models.json` `base_url` override, which bypasses the gateway's own upstream-timeout assumption entirely | diff --git a/crates/agent/scripts/scan-release-seams.sh b/crates/agent/scripts/scan-release-seams.sh new file mode 100755 index 00000000..51e8bd26 --- /dev/null +++ b/crates/agent/scripts/scan-release-seams.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Fail if any given build artifact carries a test seam's name. +# +# A test seam (`BEYOND_AI_AGENT_TEST_SLOW_CHECKPOINT_MS`, ...) changes how the binary behaves when an +# environment variable is set: fine in a debug test binary, never in a shipped one. Every seam is +# compiled only under `#[cfg(debug_assertions)]`, so a release binary must not contain the prefix at +# all. This scans the artifact itself: whatever the source looks like (a name assembled with +# `concat!`, one the source checker in `tests/release_seams.rs` misreads), a name the compiler kept +# is a contiguous string in the binary, and this finds it. CI runs it on every release binary the +# `build (release)` job produces (`mise check:release-seams`). +# +# Usage: scan-release-seams.sh ... +set -eu +if [ "$#" -eq 0 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi +status=0 +for artifact in "$@"; do + if grep -aq 'BEYOND_AI_AGENT_TEST_' "$artifact"; then + echo "$artifact carries test seam names (they must exist only in debug builds):" >&2 + grep -ao 'BEYOND_AI_AGENT_TEST_[A-Z0-9_]*' "$artifact" | sort -u | sed 's/^/ /' >&2 + status=1 + fi +done +exit "$status" diff --git a/crates/agent/tests/release_seams.rs b/crates/agent/tests/release_seams.rs index 2ef3c6f2..4072fa74 100644 --- a/crates/agent/tests/release_seams.rs +++ b/crates/agent/tests/release_seams.rs @@ -5,7 +5,13 @@ //! shipped one. Each lives in a function (or a match arm) under `#[cfg(debug_assertions)]`, with a //! no-op twin for release, so a release binary has no injection path and carries none of the names. //! A name written at a call site instead — `delay("BEYOND_AI_AGENT_TEST_…")` into a gated helper — -//! still compiles into release as a string the binary carries; this test fails on that. +//! still compiles into release as a string the binary carries. +//! +//! **The authority is the release binary itself**: CI's `build (release)` job runs +//! `scripts/scan-release-seams.sh` on every release binary it built (`mise check:release-seams`), +//! and fails on any `BEYOND_AI_AGENT_TEST_` string the compiler kept, however the source spelled it +//! (`concat!`, a `//` inside a string literal, anything a text heuristic misreads). The source check +//! here is only the fast early warning; the scanner's own behaviour is pinned below. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] /// Where a seam name may appear: in a function, or a match arm, marked `#[cfg(debug_assertions)]`. @@ -146,3 +152,40 @@ async fn body() { "; assert_eq!(ungated_seam_names("f.rs", gated_neighbour).len(), 1); } + +/// The artifact scanner CI trusts: it fails on a binary that carries seam names — this test's own +/// debug build of the agent does, by design — and passes one that does not. A name assembled at +/// compile time (`concat!`) is a contiguous string in the artifact, so it is caught the same way. +#[test] +fn the_release_artifact_scan_fails_on_any_seam_name() { + let script = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("scripts/scan-release-seams.sh"); + let scan = |artifact: &std::path::Path| { + std::process::Command::new("sh") + .arg(&script) + .arg(artifact) + .output() + .unwrap() + }; + let debug = scan(std::path::Path::new(env!("CARGO_BIN_EXE_beyond-ai-agent"))); + assert_eq!(debug.status.code(), Some(1), "{debug:?}"); + assert!( + String::from_utf8_lossy(&debug.stderr).contains("BEYOND_AI_AGENT_TEST_SLOW_CHECKPOINT_MS"), + "{debug:?}" + ); + let dir = tempfile::tempdir().unwrap(); + let clean = dir.path().join("clean"); + std::fs::write( + &clean, + b"\x7fELF BEYOND_AI_AGENT_MCP_IDLE_SECS nothing to see", + ) + .unwrap(); + assert_eq!(scan(&clean).status.code(), Some(0)); + let assembled = dir.path().join("assembled"); + std::fs::write( + &assembled, + concat!("\x7fELF ", "BEYOND_AI_AGENT_", "TEST_", "SLOW_X_MS").as_bytes(), + ) + .unwrap(); + assert_eq!(scan(&assembled).status.code(), Some(1)); +} diff --git a/mise.toml b/mise.toml index e28a8243..97da9d75 100644 --- a/mise.toml +++ b/mise.toml @@ -18,6 +18,10 @@ run = "cargo build" [tasks."build:rs:release"] run = "cargo build --release" +[tasks."check:release-seams"] +description = "Fail if a release binary carries a test seam's name (run after build:rs:release)." +run = "crates/agent/scripts/scan-release-seams.sh $(find target/release -maxdepth 1 -type f -perm -u+x)" + [tasks."ai:mint-dev-key"] description = "Mint a deterministic DEV-ONLY signing pubkey + bai_v1 token (fixed seed) for wiring up the dev config. Not for production." run = "cargo run -p beyond-ai --example mint_dev_key"