Run the open-loop scale benchmark over the corpus

.claude/CLAUDE.md

@@ -834,14 +834,17 @@ then on "Custom firecracker not found", because a hand-rolled runner called
 | `bench-chromium-hostcdp` | `bench-chromium-request-build` | host-container CDP baseline, no VM; `COMPARISON_LABEL=` (default `standalone`), `CPU_BUDGET=` (default `unlimited`), `CPUS=` (requires `CPU_BUDGET=vm-matched`), `BENCH_RESOLVE_ALL_TO=` |
 | `bench-chromium-fault` | `build` + `setup-default` | `FAULT_OUT=` (required), `FAULT_ARGS=` |
 
-- **verify/diag/run must never gain a `build` dependency.** reqbench.sh seals
-  fcvm, fc-agent and its six sources into a hash-bound runtime bundle; the
-  run refuses a golden recorded under a different bundle hash. A rebuild, an
+- **verify/diag/run and bench-chromium-scale must never gain a `build` dependency.**
+  reqbench.sh seals fcvm, fc-agent, the request harness's six sources and the
+  scale harness's five into a hash-bound runtime bundle; the run, and the scale
+  run through `reqbench.sh scale`, refuse a golden recorded under a different
+  bundle hash. A rebuild, an
   edit to a sealed file, or a new commit between golden and run invalidates
   the chain: the run also refuses a golden whose recorded source_revision is
   not the current git HEAD. Regolden instead of working around the seal.
   Sealed set: reqbench.{sh,py}, reqanalyze.py, cdpdrive.py, render.py,
-  wddrive.py, fcvm, fc-agent. The Makefile and test files are not sealed:
+  wddrive.py, reqscale.py, reqscale_analyze.py, faulttrace.bt, guardexec.py,
+  guardsupervise.py, fcvm, fc-agent. The Makefile and test files are not sealed:
   uncommitted edits to them do not affect a golden or a run in progress, but
   committing them before the run does.
 - Hugepage goldens are part of the snapshot identity: distinct tag, e.g.

Makefile

@@ -1184,8 +1184,15 @@ test-chromium-fault:
 #
 # Every cell and every publication gate is required: an accidental benchmark is
 # worse than no benchmark, so there are no defaults to fall back on.
-bench-chromium-scale: private SHELL := $(TARGET_LEASE_SHELL)
-bench-chromium-scale: build
+# The memory server's mode for the UFFD backend, and the host control's page.
+# SCALE_URL may be a comma-separated list (the corpus); then SCALE_CONTROL_URL
+# names the one page the control renders, and SCALE_CONTROL_RESOLVE_ALL_TO maps
+# the control's names to a replay server.
+SCALE_UFFD_MODE ?= copy
+SCALE_UFFD_PREFETCH ?= on
+SCALE_CONTROL_URL ?=
+SCALE_CONTROL_RESOLVE_ALL_TO ?=
+bench-chromium-scale:
 	@test -n "$(SCALE_RATES)" || (echo "ERROR: SCALE_RATES required (for example 2,4,8)"; exit 1)
 	@test -n "$(SCALE_BURSTS)" || (echo "ERROR: SCALE_BURSTS required (must be at least 5)"; exit 1)
 	@test -n "$(SCALE_SEED)" || (echo "ERROR: SCALE_SEED required"; exit 1)
@@ -1199,8 +1206,7 @@ bench-chromium-scale: build
 	@test -n "$(SCALE_MAX_LAUNCH_LAG_MS)" || (echo "ERROR: SCALE_MAX_LAUNCH_LAG_MS required"; exit 1)
 	@test -n "$(SCALE_MAX_CONTROL_DRIFT_PCT)" || (echo "ERROR: SCALE_MAX_CONTROL_DRIFT_PCT required"; exit 1)
 	@echo "==> Running open-loop Chromium request scalability benchmark..."
-	sudo -E env RUST_LOG=fcvm=debug python3 bench/chromium/reqscale.py \
-		--fcvm ./target/release/fcvm --snapshot-tag "$(SCALE_TAG)" \
+	TAG="$(SCALE_TAG)" RESULTS="$(RESULTS)" bash bench/chromium/reqbench.sh scale \
 		--url "$(SCALE_URL)" --rates "$(SCALE_RATES)" \
 		--bursts "$(SCALE_BURSTS)" --control-chromium "$(SCALE_CONTROL_CHROMIUM)" \
 		--max-offered-rps-error-pct "$(SCALE_MAX_OFFERED_ERROR_PCT)" \
@@ -1209,6 +1215,9 @@ bench-chromium-scale: build
 		--max-p95-launch-lag-ms "$(SCALE_MAX_LAUNCH_LAG_MS)" \
 		--max-control-median-drift-pct "$(SCALE_MAX_CONTROL_DRIFT_PCT)" \
 		--seed "$(SCALE_SEED)" \
+		--uffd-mode "$(SCALE_UFFD_MODE)" --uffd-prefetch "$(SCALE_UFFD_PREFETCH)" \
+		$(if $(SCALE_CONTROL_URL),--control-url "$(SCALE_CONTROL_URL)") \
+		$(if $(SCALE_CONTROL_RESOLVE_ALL_TO),--control-resolve-all-to "$(SCALE_CONTROL_RESOLVE_ALL_TO)") \
 		--out-dir "$(SCALE_OUT)" $(SCALE_TRACE_ARGS)
 
 analyze-chromium-scale:

bench/chromium/AGENTS.md

@@ -199,8 +199,9 @@ hostname URLs and records no resolver (`guest_dns` null, no
 2026-08-16 corpus record, whose guests resolved through ambient DNS. A marker writer must first open the results directory and
 take an exclusive `flock` on that directory descriptor, then remove completion
 records and atomically rename the marker while holding the lock.
-`campaign_summary.py` and `compare.py` hold shared locks on every run directory
-whose authorization they consume, across validation and publication. This
+`campaign_summary.py`, `compare.py` and `reqscale_analyze.py` hold shared locks
+on every run directory whose authorization they consume, across validation and
+publication. This
 makes a withdrawal order before the publication (which refuses it) or after it
 (which invalidates it), with no unchecked interval between the last marker read
 and publication. The marker is tracked (`!results/**/WITHDRAWN` in

bench/chromium/corpus_campaign.sh

@@ -77,9 +77,35 @@ done
 # DIAG_ONLY=1 ends the campaign after golden, verify and diag: no measured
 # run, no analysis. For the throwaway golden round.
 DIAG_ONLY="${DIAG_ONLY:-0}"
+# MEASURE=scale replaces the serial reqbench run with the open-loop throughput
+# benchmark (make bench-chromium-scale) over the same corpus, inside the same
+# golden, verify and DNS-evidence bracket. Its rates, bursts, seed, gates and
+# control Chromium come from the caller's SCALE_* variables; the corpus URLs,
+# tag, memory-server mode and output directory come from this campaign. The
+# host control renders SCALE_CONTROL_URL (default: the corpus's first URL)
+# with every name mapped to this host's replay server.
+MEASURE="${MEASURE:-reqbench}"
+case "$MEASURE" in
+    reqbench) ;;
+    scale)
+        [ "${ENGINE:-chromium}" = chromium ] \
+            || { echo "BLOCKED: MEASURE=scale drives Chromium only (ENGINE=$ENGINE)" >&2; exit 2; }
+        [ "$BACKEND" = uffd ] \
+            || { echo "BLOCKED: MEASURE=scale pairs FILE and UFFD restores itself; BACKEND=$BACKEND has no meaning there" >&2; exit 2; }
+        ;;
+    *) echo "BLOCKED: MEASURE must be reqbench or scale (got '$MEASURE')" >&2; exit 2 ;;
+esac
 STAMP="$(date +%Y%m%d-%H%M%S)"
 RESULTS="${RESULTS:-$REPO/bench/chromium/results/reqbench-$STAMP-corpus}"
 LOGDIR="${LOGDIR:-/tmp/corpus-campaign-$STAMP}"
+# The analysis the DNS evidence names its run from: the serial run's own, or
+# the scale run's first analysis (the one that may publish comes after the
+# evidence exists).
+if [ "$MEASURE" = scale ]; then
+    ANALYSIS_JSON="$RESULTS/scale/analysis-pre-evidence.json"
+else
+    ANALYSIS_JSON="$RESULTS/analysis.json"
+fi
 mkdir -p "$LOGDIR"
 # Created here, not left to reqbench: the replay server's logs and the resolver
 # evidence below are written into it before any reqbench phase runs.
@@ -128,6 +154,14 @@ acquire_diag_lock || exit 2
 # sub-make, and an earlier campaign's passed=true must not answer for this
 # one. The content-addressed runtime bundles under runtime/ and the phase
 # logs under logs/ are not the record and stay.
+# An earlier scale attempt's directory goes too: reqscale.py refuses to reuse
+# one. reqbench.sh scale hands it back to this user, but a run killed before
+# that leaves it owned by root, so sudo removes only what this user cannot,
+# with the diag lock's descriptor closed so no helper of sudo inherits it.
+if [ -e "${RESULTS:?}/scale" ]; then
+    rm -rf -- "$RESULTS/scale" 2>/dev/null \
+        || sudo rm -rf -- "$RESULTS/scale" {CAMPAIGN_DIAG_LOCK_FD}>&-
+fi
 rm -f "$RESULTS"/dns-evidence.json "$RESULTS"/verify-dns*.json "$RESULTS"/dns-owner.log \
     "$RESULTS"/replay-queries.log \
     "$RESULTS"/corpus-dns.log "$RESULTS"/corpus-access.log "$RESULTS"/corpus-serve.status \
@@ -484,6 +518,7 @@ write_dns_evidence() {
     # of this box names nothing; campaign_summary resolves each name beside
     # the record, and verify_file_sha256 is keyed by the same names.
     local verdict="$1" reason="${2:-}" owner_log="dns-owner.log"
+    local ANALYSIS_JSON="${ANALYSIS_JSON:-$RESULTS/analysis.json}"
     local log="$RESULTS/$owner_log" out="$RESULTS/dns-evidence.json"
     local samples=0 first_mismatch="" before=false after=false after_state f
     local sampler_alive=false load_stats load_samples=0 load_max=null
@@ -496,13 +531,13 @@ write_dns_evidence() {
     # ago; reqbench.py stamps that id into every record, so it survives a
     # re-analysis, which a hash of analysis.json would not.
     local run_id=""
-    if [ -f "$RESULTS/analysis.json" ]; then
+    if [ -f "$ANALYSIS_JSON" ]; then
         run_id=$(jq -r 'select((.run_id | type) == "string" and (.run_id | length) > 0)
-                        | .run_id' "$RESULTS/analysis.json" 2>/dev/null) || run_id=""
+                        | .run_id' "$ANALYSIS_JSON" 2>/dev/null) || run_id=""
     fi
     if [ -z "$run_id" ]; then
         verdict=unclean
-        reason="${reason:-$RESULTS/analysis.json names no run_id, so this evidence is bound to no measured run}"
+        reason="${reason:-$ANALYSIS_JSON names no run_id, so this evidence is bound to no measured run}"
     fi
     [ "$DNSMASQ_WAS_ACTIVE" = yes ] && before=true
     [ "$SAMPLER_ALIVE_AT_STOP" = yes ] && sampler_alive=true
@@ -915,14 +950,33 @@ say "box quiet (1-min load $load1)"
 run_verify before-run || campaign_fail "verify (before-run) failed after the settle wait"
 
 # --- measured run ----------------------------------------------------------
-say "measured run: $REPS reps/arm, warmup $WARMUP, arms $ARMS, $BACKEND/$UFFD_MODE prefetch=$UFFD_PREFETCH"
-start_dns_sampler
 run_rc=0
-TAG="$TAG" URL="$URLS" BACKEND="$BACKEND" UFFD_MODE="$UFFD_MODE" \
-    UFFD_PREFETCH="$UFFD_PREFETCH" ARMS="$ARMS" REPS="$REPS" WARMUP="$WARMUP" \
-    STALL_MAX_MS="$STALL_MAX_MS" RESULTS="$RESULTS" ENGINE="$ENGINE" \
-    make -C "$REPO" "$(engine_target run)" 2>&1 | tee "$LOGDIR/run.log" || run_rc=$?
-stop_dns_sampler
+if [ "$MEASURE" = scale ]; then
+    say "measured run: open-loop scale over the corpus, rates ${SCALE_RATES:-unset}, $UFFD_MODE prefetch=$UFFD_PREFETCH"
+    start_dns_sampler
+    SCALE_URL="$URLS" SCALE_TAG="$TAG" SCALE_OUT="$RESULTS/scale" \
+        SCALE_UFFD_MODE="$UFFD_MODE" SCALE_UFFD_PREFETCH="$UFFD_PREFETCH" \
+        SCALE_CONTROL_URL="${SCALE_CONTROL_URL:-${URLS%%,*}}" \
+        SCALE_CONTROL_RESOLVE_ALL_TO="${SCALE_CONTROL_RESOLVE_ALL_TO:-127.0.0.1}" \
+        make -C "$REPO" bench-chromium-scale 2>&1 | tee "$LOGDIR/run.log" || run_rc=$?
+    stop_dns_sampler
+    # The analyzer writes the run id the DNS evidence below binds to, and
+    # withholds publication until that evidence exists beside the run, so it
+    # runs once here and once more after the evidence is written.
+    if [ "$run_rc" -eq 0 ]; then
+        make -C "$REPO" analyze-chromium-scale SCALE_RUN_DIR="$RESULTS/scale" \
+            SCALE_ANALYSIS_JSON="$ANALYSIS_JSON" 2>&1 | tee "$LOGDIR/analyze.log" \
+            || run_rc=$?
+    fi
+else
+    say "measured run: $REPS reps/arm, warmup $WARMUP, arms $ARMS, $BACKEND/$UFFD_MODE prefetch=$UFFD_PREFETCH"
+    start_dns_sampler
+    TAG="$TAG" URL="$URLS" BACKEND="$BACKEND" UFFD_MODE="$UFFD_MODE" \
+        UFFD_PREFETCH="$UFFD_PREFETCH" ARMS="$ARMS" REPS="$REPS" WARMUP="$WARMUP" \
+        STALL_MAX_MS="$STALL_MAX_MS" RESULTS="$RESULTS" ENGINE="$ENGINE" \
+        make -C "$REPO" "$(engine_target run)" 2>&1 | tee "$LOGDIR/run.log" || run_rc=$?
+    stop_dns_sampler
+fi
 
 # The run's own exit is not the verdict: a run that measured cleanly against
 # the wrong resolver is worse than one that failed, so the after-run bracket
@@ -937,6 +991,17 @@ if [ "$run_rc" -ne 0 ] || [ "$after_rc" -ne 0 ] || [ "$verdict" != clean ]; then
     echo "FAILED: measured run exit $run_rc, after-run verify exit $after_rc, dns verdict $verdict" >&2
     exit 1
 fi
+if [ "$MEASURE" = scale ]; then
+    make -C "$REPO" analyze-chromium-scale SCALE_RUN_DIR="$RESULTS/scale" \
+        SCALE_ANALYSIS_JSON="$RESULTS/scale/analysis.json" 2>&1 \
+        | tee -a "$LOGDIR/analyze.log" \
+        || { echo "FAILED: the scale analysis after the DNS evidence did not run" >&2; exit 1; }
+    jq -e '.publishable == true' "$RESULTS/scale/analysis.json" >/dev/null || {
+        echo "FAILED: scale analysis not publishable: $(jq -r '.publication_blocked_by' "$RESULTS/scale/analysis.json" 2>&1)" >&2
+        exit 1
+    }
+    say "scale analysis: $RESULTS/scale/analysis.json (publishable)"
+fi
 
 say "records: $RESULTS"
 say "logs:    $LOGDIR"

bench/chromium/reqbench.py

@@ -1419,6 +1419,38 @@ def sha256_file(path: str) -> str:
     return h.hexdigest()
 
 
+def require_golden_runtime(snapshot_name: str, snapshot: dict, fcvm: str) -> dict:
+    """This process's staged runtime, refused unless it created the golden.
+
+    The runtime is the reqbench.sh bundle this process executes from: the
+    fcvm it runs, the hash of the bundle's MANIFEST.sha256, and the revision
+    it was staged at. reqbench.sh golden records the same three values for
+    the bundle that created the snapshot (snapshot_generation returns them).
+    """
+    runtime_bundle = os.environ.get("REQBENCH_RUNTIME_BUNDLE", "")
+    manifest_path = os.path.join(runtime_bundle, "MANIFEST.sha256")
+    if not runtime_bundle or os.path.realpath(runtime_bundle) != os.path.realpath(HERE):
+        raise RuntimeError(
+            f"{HERE} is not reqbench.sh's staged runtime bundle "
+            f"(REQBENCH_RUNTIME_BUNDLE={runtime_bundle!r}); measured phases run from it"
+        )
+    if not os.path.isfile(manifest_path):
+        raise RuntimeError(f"staged runtime manifest is missing: {manifest_path}")
+    runtime = {
+        "creator_fcvm_sha256": sha256_file(fcvm),
+        "creator_runtime_bundle_sha256": sha256_file(manifest_path),
+        "source_revision": os.environ.get("REQBENCH_SOURCE_REVISION", ""),
+    }
+    for field, current in runtime.items():
+        if snapshot[field] != current:
+            raise RuntimeError(
+                f"snapshot {snapshot_name} was created with {field}="
+                f"{snapshot[field]!r}, current runtime is {current!r}; recreate "
+                "the golden with this staged runtime"
+            )
+    return runtime
+
+
 # Every script that defines one request sample, for either engine. Must stay
 # equal to reqbench.sh's staged runtime sources minus the binaries and the
 # analyzer (which reads samples but defines none); asserted by
@@ -4398,27 +4430,13 @@ def main_with_resources(resources: ExitStack) -> int:
         p.error(str(error))
 
     args.fcvm = os.path.abspath(args.fcvm)
-    runtime_bundle = os.environ.get("REQBENCH_RUNTIME_BUNDLE", "")
-    manifest_path = os.path.join(runtime_bundle, "MANIFEST.sha256")
-    if os.path.realpath(runtime_bundle) != os.path.realpath(HERE):
-        p.error("reqbench.py must execute from reqbench.sh's staged runtime bundle")
-    if not os.path.isfile(manifest_path):
-        p.error(f"staged runtime manifest is missing: {manifest_path}")
-    current_fcvm_sha256 = sha256_file(args.fcvm)
-    current_runtime_bundle_sha256 = sha256_file(manifest_path)
-    current_source_revision = os.environ.get("REQBENCH_SOURCE_REVISION", "")
-    creator_identity = {
-        "creator_fcvm_sha256": current_fcvm_sha256,
-        "creator_runtime_bundle_sha256": current_runtime_bundle_sha256,
-        "source_revision": current_source_revision,
-    }
-    for field, current in creator_identity.items():
-        if snapshot[field] != current:
-            p.error(
-                f"snapshot {snapshot_name} was created with {field}="
-                f"{snapshot[field]!r}, current runtime is {current!r}; recreate "
-                "the golden with this staged runtime"
-            )
+    try:
+        runtime = require_golden_runtime(snapshot_name, snapshot, args.fcvm)
+    except RuntimeError as error:
+        p.error(str(error))
+    current_fcvm_sha256 = runtime["creator_fcvm_sha256"]
+    current_runtime_bundle_sha256 = runtime["creator_runtime_bundle_sha256"]
+    current_source_revision = runtime["source_revision"]
     os.makedirs(args.out_dir, exist_ok=True)
     arms = [a.strip() for a in args.arms.split(",") if a.strip()]
     allowed_arms = allowed_arms_for_engine(args.engine)

bench/chromium/reqbench.sh

@@ -124,18 +124,18 @@ if [ "${BASH_SOURCE[0]}" = "$0" ] && [ "${REQBENCH_STAGED:-0}" != 1 ]; then
     source_revision_before=$(git -C "$REPO" rev-parse HEAD)
     mkdir -p "$RESULTS/runtime"
     stage_dir=$(mktemp -d "$RESULTS/runtime/.stage.XXXXXX")
-    for source in reqbench.sh reqbench.py reqanalyze.py cdpdrive.py render.py wddrive.py; do
+    # The request harness, then the scale harness (reqscale.HARNESS_SOURCES).
+    runtime_sources=(reqbench.sh reqbench.py reqanalyze.py cdpdrive.py render.py wddrive.py
+        reqscale.py reqscale_analyze.py faulttrace.bt guardexec.py guardsupervise.py)
+    for source in "${runtime_sources[@]}"; do
         cp --reflink=auto "$HERE/$source" "$stage_dir/$source"
     done
     cp --reflink=auto "$FC_AGENT" "$stage_dir/fc-agent"
     cp --reflink=auto "$FCVM" "$stage_dir/fcvm"
-    chmod 0555 "$stage_dir/fcvm" "$stage_dir/fc-agent" "$stage_dir/reqbench.sh" \
-        "$stage_dir/reqbench.py" "$stage_dir/reqanalyze.py" \
-        "$stage_dir/cdpdrive.py" "$stage_dir/render.py" "$stage_dir/wddrive.py"
     (
         cd "$stage_dir"
-        sha256sum fcvm fc-agent reqbench.sh reqbench.py reqanalyze.py cdpdrive.py render.py wddrive.py \
-            > MANIFEST.sha256
+        chmod 0555 fcvm fc-agent "${runtime_sources[@]}"
+        sha256sum fcvm fc-agent "${runtime_sources[@]}" > MANIFEST.sha256
     )
     bundle_hash=$(sha256sum "$stage_dir/MANIFEST.sha256" | cut -d' ' -f1)
     bundle_dir="$RESULTS/runtime/$bundle_hash"
@@ -2044,6 +2044,29 @@ cmd_diag() {
     return $rc
 }
 
+# The open-loop scale benchmark, from this bundle like every measured phase:
+# reqscale.py refuses a golden created by another bundle, fcvm or revision.
+cmd_scale() {
+    local rc=0 out="" prev=""
+    for arg in "$@"; do
+        [ "$prev" = --out-dir ] && out=$arg
+        prev=$arg
+    done
+    sudo -E env RUST_LOG=fcvm=debug \
+        REQBENCH_RUNTIME_BUNDLE="${REQBENCH_RUNTIME_BUNDLE:-}" \
+        REQBENCH_SOURCE_REPO="$REPO" \
+        REQBENCH_SOURCE_REVISION="${REQBENCH_SOURCE_REVISION:-}" \
+        python3 "$HERE/reqscale.py" --snapshot-tag "$TAG" \
+        --data-root "$DATA_ROOT" --state-dir "$STATE_DIR" "$@" || rc=$?
+    # reqscale.py runs as root and creates its output as root; the analysis and
+    # the campaign's evidence step run as this user and write into it.
+    if [ -n "$out" ] && [ -e "$out" ]; then
+        sudo chown -R "$(id -u):$(id -g)" -- "$out" || rc=1
+    fi
+    verify_runtime_bundle || rc=1
+    return $rc
+}
+
 # Only dispatch when EXECUTED. Sourcing the file makes its helpers unit-testable
 # (see ReqbenchShell in test_reqbench.py) instead of reachable only through a
 # whole phase.
@@ -2055,13 +2078,14 @@ if [ "${BASH_SOURCE[0]}" = "$0" ]; then
         verify) cmd_verify ;;
         run)    cmd_run ;;
         diag)   cmd_diag ;;
+        scale)  shift; cmd_scale "$@" ;;
         all)
             # The chain's own build/golden/verify phases are the load the run
             # gate reads a minute later; default the settle window so a cold
             # one-shot does not refuse on its own wake. An explicit
             # SETTLE_WAIT_SECS wins.
             export SETTLE_WAIT_SECS="${SETTLE_WAIT_SECS:-120}"
             cmd_build; cmd_golden; cmd_verify; cmd_run ;;
-        *) echo "usage: $0 {build|golden|verify|run|diag|all}" >&2; exit 2 ;;
+        *) echo "usage: $0 {build|golden|verify|run|diag|scale|all}" >&2; exit 2 ;;
     esac
 fi