diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml
index aa066bbd5..23e323d57 100644
--- a/.github/workflows/bench-abba.yml
+++ b/.github/workflows/bench-abba.yml
@@ -1,18 +1,37 @@
name: Bench ABBA tiebreaker
-# Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the
-# cheap PR benchmark can't confirm. At the default workload it OCCUPIES THE SINGLE
-# BENCH SERVER FOR SEVERAL HOURS, so it NEVER auto-triggers -- it runs only on an
-# explicit `/bench-abba` comment on a PR.
+# Drift-free paired (A/B/B/A) prover benchmark for resolving the small deltas /bench
+# cannot confirm. It proves the SAME real block /bench does, so a 3-10% reading there
+# escalates to a paired test of the same workload rather than a different one. It
+# OCCUPIES THE SINGLE BENCH SERVER FOR OVER AN HOUR, so it NEVER auto-triggers -- it
+# runs only on an explicit `/bench-abba` comment on a PR.
#
-# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (default: 20 pairs, ethrex 100tx
-# --continuations; "cont10" is the quick coarse option, "mono[TX]" the legacy
-# monolithic prove). Cont proofs need rkyv pointer_width_64 on both sides, so PR
-# branches older than that fix must rebase before a cont bench.
+# Syntax: "/bench-abba [N]", where N is the pair count (default 12).
+#
+# Resolution, from the paired t-test (resolvable 95% delta = t* x sd / sqrt(N)). The
+# pair-delta sd on this runner is NOT yet measured; the two columns bracket it between
+# 1.0% (the GPU box's measured 0.64% plus margin) and 2.0% (sqrt(2) x this runner's
+# measured 1.43% single-run CV):
+#
+# pairs wall resolves (sd 2.0% / sd 1.0%)
+# 8 ~50 min 1.7% / 0.8%
+# 12 ~72 min 1.3% / 0.6% <- default
+# 20 ~1h55m 0.9% / 0.5%
+# 32 ~3h 0.7% / 0.4%
+#
+# Wall is 2 x 158.8 s per pair (the runner's measured prove at epoch 2^22) plus ~8 min
+# of setup. The FIRST run measures the sd — it is the `sd` field of the paired-t line
+# in the result comment — and this table should be re-pinned to it then.
on:
issue_comment:
types: [created]
+env:
+ # The epoch /bench proves this block at, so the tiebreaker resolves the same
+ # configuration and not just the same block. Memory picks it: this runner peaks at
+ # ~52 GB here against its >=64 GiB floor. See benchmark-pr.yml's REAL_BLOCK_EPOCH_LOG2.
+ ABBA_REAL_EPOCH_LOG2: "22"
+
concurrency:
# See bench-verify.yml. The single self-hosted bench runner serializes across PRs.
group: ${{ startsWith(github.event.comment.body, '/bench-abba') && format('bench-abba-{0}', github.event.issue.number) || format('bench-abba-ignore-{0}', github.run_id) }}
@@ -31,9 +50,10 @@ jobs:
startsWith(github.event.comment.body, '/bench-abba') &&
contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association)
runs-on: [self-hosted, bench]
- # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the
- # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds.
- timeout-minutes: 720
+ # Hang guardrail, not expected duration: a real-block pair is 2 x 158.8 s = ~5.3 min,
+ # so the default 12 pairs runs ~72 min and the 40-pair clamp ~3.7 hr, plus up to
+ # ~30 min of two-sided build on a cold cache.
+ timeout-minutes: 360
steps:
- name: Resolve PR head + bench config
id: cfg
@@ -50,76 +70,38 @@ jobs:
# force-push race mid-run.
HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid)
echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT"
- # Everything after "/bench-abba" on its line, tokens in any order: a number =
- # pair count; cont[TX]/mono[TX] = workload.
+ # Everything after "/bench-abba" on its line. The only token is an optional
+ # pair count; the workload is fixed to the real block.
ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1)
- PAIRS=20; CONTINUATIONS=1; TX_COUNT=100
+ PAIRS=12
set -f # tokens must not glob-expand against the runner's CWD
for tok in $ARGS; do
case "$tok" in
- cont) CONTINUATIONS=1; TX_COUNT=100 ;;
- mono) CONTINUATIONS=0; TX_COUNT=5 ;;
- cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;;
- mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;;
- [0-9]*) PAIRS="$tok" ;;
- *) echo "::warning::ignoring unrecognized token '$tok'" ;;
+ [0-9]*) PAIRS="$tok" ;;
+ *) echo "::warning::ignoring unrecognized token '$tok'" ;;
esac
done
- # Digits-only + clamps: PAIRS capped so one comment can't monopolize the
- # single bench server; mono capped at 5tx (monolithic peak heap grows with
- # the trace; 20tx needs ~78 GB).
- if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi
- case "$TX_COUNT" in
- ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;;
- esac
- if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then
- echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT"
- TX_COUNT=$TX_DEFAULT
- fi
+ # Digits-only + clamp, so one comment cannot monopolize the single bench
+ # server: 40 pairs is already ~3.7 hr of it.
case "$PAIRS" in
- ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 20"; PAIRS=20 ;;
+ ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 12"; PAIRS=12 ;;
esac
if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 40 ]; then
- echo "::warning::pair count $PAIRS out of range [2,40]; using 20"
- PAIRS=20
+ echo "::warning::pair count $PAIRS out of range [2,40]; using 12"
+ PAIRS=12
fi
# Even is ideal so the AB/BA orders balance; round an odd request up by one.
if [ "$((PAIRS % 2))" -ne 0 ]; then
PAIRS=$((PAIRS + 1))
echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance"
fi
- if [ "$CONTINUATIONS" = "1" ]; then
- WORKLOAD="ethrex ${TX_COUNT}tx continuations"
- else
- WORKLOAD="ethrex ${TX_COUNT}tx monolithic"
- fi
- # Outputs land before the fail-fast below so the always() result
- # comment is fully labeled even when this step exits early.
+ # The block's identity lives in the Makefile; this job never names one.
+ WORKLOAD="ethrex real block, continuations"
{
echo "pairs=$PAIRS"
- echo "continuations=$CONTINUATIONS"
- echo "tx_count=$TX_COUNT"
echo "workload=$WORKLOAD"
} >> "$GITHUB_OUTPUT"
- # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx
- # continuation proof exceeds rkyv's old 2 GiB cap and only dies after
- # blocking the single bench server for hours. Skip the check when the
- # fetch fails: gh api prints HTTP error bodies to stdout, so gate on
- # its exit status AND on the payload looking like the manifest (it
- # declares rkyv) — never treat an error blob as a missing feature.
- # Purely textual; deletable once every open branch postdates the fix.
- if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then
- if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \
- -H "Accept: application/vnd.github.raw" 2>/dev/null) \
- && printf '%s' "$SIDE" | grep -q '^rkyv' \
- && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then
- MSG="PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono."
- echo "$MSG" > /tmp/abba_out.txt # surfaces in the result comment
- echo "::error::$MSG"
- exit 1
- fi
- fi
- echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD"
+ echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD at epoch 2^$ABBA_REAL_EPOCH_LOG2"
- name: Acknowledge (react + occupancy notice)
uses: actions/github-script@v7
@@ -135,7 +117,9 @@ jobs:
await github.rest.issues.createComment({
owner: context.repo.owner, repo: context.repo.repo,
issue_number: context.issue.number,
- body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} (a cont100 pair is ~15 min, so the default 20 pairs runs ~4.5-5 hr; pass a smaller pair count or \`cont10\` for a quicker, coarser run). The bench server is occupied until it finishes.`
+ // A pair is TWO proves at the runner's measured 158.8 s, so ~5.3 min/pair,
+ // plus ~8 min of checkout, two-sided build and fixture fetch.
+ body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} — a pair is 2 proves at ~158.8 s, so roughly ${Math.round(8 + Number(process.env.PAIRS) * 5.3)} min. Pass a smaller pair count for a quicker, coarser run. The bench server is occupied until it finishes.`
});
- name: Checkout (full history for ref resolution)
@@ -156,8 +140,12 @@ jobs:
env:
HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
PAIRS: ${{ steps.cfg.outputs.pairs }}
- CONTINUATIONS: ${{ steps.cfg.outputs.continuations }}
- TX_COUNT: ${{ steps.cfg.outputs.tx_count }}
+ # WORKLOAD=real is bench_abba.sh's own default; pinned here so the workload
+ # this job measures is stated in the job rather than inherited. The epoch is
+ # NOT the script's default (2^20, laptop-safe) — it is the runner's tier, the
+ # same one /bench proves at.
+ WORKLOAD: real
+ EPOCH_SIZE_LOG2: ${{ env.ABBA_REAL_EPOCH_LOG2 }}
run: |
export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
set -o pipefail
diff --git a/.github/workflows/bench-verify.yml b/.github/workflows/bench-verify.yml
index 23d616ec4..7774e0a6c 100644
--- a/.github/workflows/bench-verify.yml
+++ b/.github/workflows/bench-verify.yml
@@ -116,6 +116,12 @@ jobs:
env:
HEAD_SHA: ${{ steps.cfg.outputs.head_sha }}
PAIRS: ${{ steps.cfg.outputs.pairs }}
+ # The script defaults to the real block; this job pins the synthetic one.
+ # /bench-verify reports TWO arms and the real path runs only the continuation
+ # one (a real block does not fit monolithically), so inheriting the default
+ # would silently drop an arm and break comparability with every verify number
+ # recorded so far.
+ WORKLOAD: synthetic
run: |
export SYSROOT_DIR="$HOME/.lambda-vm-sysroot"
set -o pipefail
diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml
index 11fcbf222..b728e60f5 100644
--- a/.github/workflows/benchmark-gpu.yml
+++ b/.github/workflows/benchmark-gpu.yml
@@ -60,9 +60,11 @@ env:
#
# GPU PATH ONLY, and deliberately so. It is NOT pushed into bench_abba.sh's default
# (2^20, which the CPU /bench-abba uses) nor into the CLI's
- # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~28 GiB of HOST memory on
- # a CPU build, which would break laptops. VRAM is the binding constraint here and host
- # RAM is the binding constraint there, so the two defaults are not the same question.
+ # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~32 GiB of HOST memory on
+ # a CPU build (measured peak RSS on the 124 GiB calibration box; the CUDA path's host
+ # peak is a different number, ~36 GB — see the cpu_ram floor below), which would break
+ # laptops. VRAM is the binding constraint here and host RAM is the binding constraint
+ # there, so the two defaults are not the same question.
GPU_REAL_EPOCH_LOG2: "22"
# Unique per-run label set on the instance, for easy identification in the Vast console.
RUN_LABEL: "gpu-bench-${{ github.run_id }}-${{ github.run_attempt }}"
@@ -157,11 +159,12 @@ jobs:
// Post the "started" notice under the SAME marker the result step uses, so the
// result updates this comment in place (and re-runs reuse it rather than stacking).
const marker = 'GPU Benchmark (ABBA)';
- // Reference: 4 pairs measured ~20 min end-to-end (rental + dual build +
- // 8 proves). Per-prove wall varies with the rented host's CPU (the prover
- // is partly host-CPU-bound), so scale expectations from pair count, not
- // from a fixed per-run figure.
- const mins = 12 + Number(process.env.PAIRS) * 2;
+ // Reference: 4 pairs measured 20 min 11 s end-to-end — 3 min 56 s of rental,
+ // checkout and dual cuda build, then 4.06 min per pair, since a pair is TWO
+ // proves at ~2 min each. Per-prove wall varies with the rented host's CPU
+ // (the prover is partly host-CPU-bound), so the slope is the measured one
+ // and the intercept carries slack for a colder box.
+ const mins = 12 + Number(process.env.PAIRS) * 4;
const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. Rough ETA ~${mins} min. The result will replace this comment.`;
const comments = await github.paginate(github.rest.issues.listComments, {
owner: context.repo.owner, repo: context.repo.repo,
diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml
index f9c83ea3b..cd72025a6 100644
--- a/.github/workflows/benchmark-pr.yml
+++ b/.github/workflows/benchmark-pr.yml
@@ -12,12 +12,13 @@ on:
- 'executor/**'
- 'bin/cli/**'
- 'tooling/ethrex-fixtures/**'
- # A baseline is only valid for the workload it measured, and these two
- # define the real-block workload: the converter decides the fixture's
- # bytes, the Makefile decides which block it is. Without them a repointed
- # block would leave main's baseline stale until some prover file happened
- # to change — and the comparison guard would suppress the table until then.
- - 'tooling/ethrex-block-converter/**'
+ # A baseline is only valid for the workload it measured, and the Makefile is
+ # what defines that workload: it names the block and pins the URL and sha256
+ # of the .bin this job fetches. Without it a repointed block would leave
+ # main's baseline stale until some prover file happened to change — and the
+ # comparison guard would suppress the table until then. The converter is not
+ # listed: CI never runs it, so changing it cannot change the bytes proven
+ # here.
- 'Makefile'
# Uncomment to auto-run on PRs:
# pull_request:
@@ -46,8 +47,8 @@ env:
# 1. Its only unique coverage was the MONOLITHIC prove path, which is vestigial
# (reportedly slower than a 1-epoch continuation). Coverage of a path we intend
# to delete is not a reason to spend runner time.
- # 2. Its crypto mix is one no real block has: 8.83 ECSM per Mcycle against a real
- # block's ~1.3-1.6. Screening against it tunes the prover for a worst case that
+ # 2. Its crypto mix is one no real block has: 9.16 ECSM per Mcycle against a real
+ # block's 2.28. Screening against it tunes the prover for a worst case that
# cannot occur.
#
# The synthetic fixtures themselves are NOT gone — /bench-growth still sweeps them to
@@ -90,28 +91,31 @@ env:
# 2^22 397.88 s 32.21 GiB RSS 13 epochs 1.15 GB proof
# 2^23 356.47 s 60.01 GiB RSS 7 epochs 0.90 GB proof
#
- # This runner's floor is >=64 GiB, so 2^22's 32.2 GiB fits with ~50% headroom while
- # 2^23's 60 GiB does not fit safely — memory, not speed, is what picks 2^22 here.
- # Moving off 2^21 is worth ~14% wall on the calibration box. That box is NOT this
- # runner (it measured 8.6 s/Mcycle against this runner's 5.3-5.6), so treat the
- # ratio as transferable and the absolute seconds as not: the first /bench run at
- # this epoch establishes this runner's actual time.
+ # Those RSS figures are the CALIBRATION BOX's, and memory does not transfer between
+ # machines any better than seconds do: this runner measured ~52 GB of peak heap for
+ # the same block at 2^22 — over 1.5x the calibration box's 32.21 GiB, and close
+ # enough to its >=64 GiB floor that 2^23 (60 GiB on the roomier box, so more here)
+ # is out of the question. Memory, not speed, is what picks 2^22. Moving off 2^21 is
+ # worth ~14% wall on the calibration box; treat that ratio as transferable and
+ # neither the absolute seconds nor the RSS column as such. This runner's own
+ # measured time at 2^22 is 158.8 s (median of 3, 2.8% spread, 13 epochs).
#
# Deliberately NOT the CLI's DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2, which stays 20 so
# a laptop can still prove; and not the GPU path's 2^22, which happens to coincide
# but is chosen by VRAM rather than host RAM (see benchmark-gpu.yml).
REAL_BLOCK_EPOCH_LOG2: "22"
- # Sampled like the synthetic screen, 3 runs, and reported the same way (median +
- # spread). Fewer than the synthetic's 5 because each run is minutes rather than
- # seconds; enough that one slow run shows up as spread instead of moving the median.
+ # Sampled 3 times and reported as median + spread. Three rather than five because
+ # each run is minutes rather than seconds; enough that one slow run shows up as
+ # spread instead of moving the median.
#
- # THIS IS THE DIAL. The runner's per-run real-block time is still unmeasured — the
- # calibration ran on a different box (see REAL_BLOCK_EPOCH_LOG2 above). If the first
- # runs land above ~6 min each, /bench becomes a ~25 min occupancy of a runner every
- # other bench queues behind, and this count is what to turn down (2, or 1) before
- # reaching for anything else.
+ # THIS IS THE DIAL. A run measures 158.8 s on this runner (median of 3, 2.8% spread,
+ # at the epoch above), so 3 runs is ~8 min of proving and the right count. Should a
+ # future block or prover change take a run past ~6 min, /bench becomes a ~25 min
+ # occupancy of a runner every other bench queues behind, and this count is what to
+ # turn down (2, or 1) before reaching for anything else.
# `/bench N` overrides this, clamped to [1,5]. Past 5 the cached comparison can't
- # beat the ~1% session-drift wall anyway — that is what /bench-abba is for.
+ # beat the ~1% session-drift wall anyway — that is what /bench-abba is for, and it
+ # proves this same block at this same epoch, so escalating keeps the question fixed.
BENCH_RUNS_REAL: 3
# Memory-scaling sweep: same ELF, different N-transfer inputs. GROWTH_PROGRAMS
# are the generated (gitignored) fixture basenames in executor/tests/; GROWTH_STEPS
@@ -210,21 +214,24 @@ jobs:
# The real block runs on EVERY invocation — plain `/bench`, push to main, and
# workflow_dispatch alike. It is the number that means something, so it should
- # not need a second command to ask for; the synthetic screen rides along
- # unchanged as the cheap, high-sensitivity half.
+ # not need a second command to ask for, and it is the only workload /bench
+ # proves.
#
- # The cost is real and lands on a shared runner: ~4.5-4.8 min per run x
- # BENCH_RUNS_REAL, so every /bench and every push to main now occupies the
- # bench server for roughly 15-20 min, and /bench-abba and /bench-verify queue
- # behind it. That trade was made deliberately — see BENCH_RUNS_REAL above for
- # the dial if it proves too expensive.
+ # The cost is real and lands on a shared runner: 158.8 s per run x
+ # BENCH_RUNS_REAL is ~8 min of proving, so every /bench and every push to main
+ # occupies the bench server for roughly 15 min once checkout, the two-sided
+ # build, the fixture fetch and the guest ELF are counted, and /bench-abba and
+ # /bench-verify queue behind it. That trade was made deliberately — see
+ # BENCH_RUNS_REAL above for the dial if it proves too expensive.
RUN_REAL=true
# The fixture is FETCHED from a pinned URL + sha256, not built (see the
# Makefile). The guard covers the window after a repoint but before the new
- # artifact is uploaded: degrade to the synthetic sections with a warning
- # rather than failing, so a push to main cannot go red over an upload nobody
- # in CI can perform.
+ # artifact is uploaded: warn instead of failing, so a push to main cannot go
+ # red over an upload nobody in CI can perform. On push and workflow_dispatch
+ # the growth sweep still runs and feeds the baseline artifact (no comment is
+ # posted on those events — the Comment step is gated to comment triggers);
+ # on a plain /bench nothing is left to measure, and the footer says so.
if [ "$RUN_REAL" = "true" ] && [ -z "$(make -s print-real-block-fixture-url)" ]; then
echo "::warning::Real-block benchmark skipped: ETHREX_REAL_BLOCK_FIXTURE_URL is unset in the Makefile."
RUN_REAL=false
@@ -237,9 +244,9 @@ jobs:
# workload even when main's Makefile names a different block.
echo "REAL_INPUT=$(make -s print-real-block-fixture)" >> "$GITHUB_ENV"
- # Every metrics writer appends; the step that used to create this file was the
- # synthetic screen, which is gone. Truncate here (this step always runs) so a
- # previous run's values can't survive on the persistent self-hosted runner.
+ # Every metrics writer appends, and no step creates this file. Truncate here
+ # (this step always runs) so a previous run's values can't survive on the
+ # persistent self-hosted runner.
: > /tmp/metrics.txt
# `/bench N` sets the real-block sample count. Both sides use the same count
@@ -253,8 +260,9 @@ jobs:
fi
# Clamp to 1-5. Beyond 5 the single-session cached comparison barely improves
- # (it can't beat the ~1% drift wall); use the ABBA tiebreaker for finer deltas.
- # At ~4-5 min a run this is also the difference between a 15 min and a 25 min
+ # (it can't beat the ~1% drift wall); use /bench-abba for finer deltas — it
+ # resolves ~1% over the same block in ~72 min at its default 12 pairs.
+ # At 158.8 s a run this is also the difference between a ~10 min and a ~20 min
# occupancy of the one bench runner.
if [ "$RUNS" -lt 1 ] 2>/dev/null || [ "$RUNS" -gt 5 ] 2>/dev/null; then
echo "::warning::Run count $RUNS out of range [1,5], defaulting to $BENCH_RUNS_REAL"
@@ -410,9 +418,9 @@ jobs:
EPOCHS="$E"
done
- # Same summary as the synthetic screen: median for the headline, spread so a
- # single slow run is visible rather than silently shifting the verdict, and
- # every raw value so a reader can judge for themselves.
+ # Median for the headline, spread so a single slow run is visible rather than
+ # silently shifting the verdict, and every raw value so a reader can judge for
+ # themselves.
MEDIAN_POS=$(( (RUNS + 1) / 2 ))
TIME_MEDIAN=$(echo $TIMES | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS")
HEAP_MEDIAN=$(echo $HEAPS | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS")
@@ -563,7 +571,7 @@ jobs:
echo "growth_r2=$R2" >> "$GITHUB_OUTPUT"
fi # end run_growth check
- # --- Real block (continuations, 1 sample) ---
+ # --- Real block (continuations, sampled to the PR side's count) ---
# Only reached when no cached baseline exists, which is the expensive case:
# push-to-main normally publishes the real-block numbers, so a /bench comment
# pays for the PR side alone. $REAL_INPUT was resolved before this
@@ -703,6 +711,19 @@ jobs:
fi
fi
+ # Growth comparison. The renderer keys the whole growth section off
+ # pr_growth_heaps, so these pass through even when only one side ran.
+ {
+ echo "pr_growth_heaps=$PR_GROWTH_HEAPS"
+ echo "pr_growth_times=$PR_GROWTH_TIMES"
+ echo "pr_growth_slope=$PR_GROWTH_SLOPE"
+ echo "pr_growth_r2=$PR_GROWTH_R2"
+ echo "baseline_growth_heaps=$BASELINE_GROWTH_HEAPS"
+ echo "baseline_growth_times=$BASELINE_GROWTH_TIMES"
+ echo "baseline_growth_slope=$BASELINE_GROWTH_SLOPE"
+ echo "baseline_growth_r2=$BASELINE_GROWTH_R2"
+ } >> "$GITHUB_OUTPUT"
+
# Growth slope comparison
if [ -n "$BASELINE_GROWTH_SLOPE" ] && [ -n "$PR_GROWTH_SLOPE" ]; then
SLOPE_DIFF=$((PR_GROWTH_SLOPE - BASELINE_GROWTH_SLOPE))
@@ -746,7 +767,7 @@ jobs:
with:
# Renderable offline: `node scripts/render_bench_comment.js` extracts this block
# and prints the markdown for a set of scenarios, so wording and formatting can
- # be checked without occupying the bench server for 15-20 min. Add a scenario
+ # be checked without occupying the bench server for ~15 min. Add a scenario
# there when you add a branch here.
script: |
const baseSrc = process.env.BASELINE_SRC;
@@ -791,8 +812,8 @@ jobs:
// --- Section 1: Real block (headline; present whenever the fixture is fetchable) ---
if (realTime) {
body += `## Benchmark — real block${realInput ? ` (\`${realInput}\`)` : ''}${parseInt(realRuns) > 1 ? ` (median of ${realRuns})` : ''}\n\n`;
- // Run count lives in the heading ("median of N"), as on the synthetic
- // section — repeating it here just made the two disagree in tone.
+ // Run count lives in the heading ("median of N"); the subheading carries
+ // only the prove configuration.
body += `continuations · epoch 2^${realEpochLog2}`;
if (realEpochs) body += ` · ${realEpochs} epochs`;
body += `\n\n`;
@@ -810,16 +831,18 @@ jobs:
}
body += `| **Prove time** | ${baseRealTime}s | ${realTime}s | ${fmt(realTimeDiff)}s (${fmt(realTimePct)}%) ${icon(realTimePct)} |\n\n`;
- // Wider bands than the synthetic screen's 5%: 3 runs of a minutes-long
- // prove resolve less than 5 runs of a 25s one, so the middle is reported
- // as unresolved rather than as "fine".
+ // Bands of 10%/3%, wider than a fast workload would need: 3 runs of a
+ // minutes-long prove resolve coarsely, so the middle is reported as
+ // unresolved rather than as "fine".
const rp = parseFloat(realTimePct);
if (rp > 10) {
body += `> ⚠️ **Regression on the real block** — prove time up ${Math.abs(rp).toFixed(1)}%.\n`;
} else if (rp < -10) {
body += `> 🎉 **Improvement on the real block** — prove time down ${Math.abs(rp).toFixed(1)}%.\n`;
} else if (Math.abs(rp) >= 3) {
- body += `> ❓ **${fmt(realTimePct)}% — beyond what ${realRuns} runs resolve.** Use \`/bench-abba\` for a paired test.\n`;
+ // /bench-abba proves this same block at this same epoch, so the
+ // escalation resolves the reading rather than changing the question.
+ body += `> ❓ **${fmt(realTimePct)}% — beyond what ${realRuns} runs resolve.** Use \`/bench-abba\` for a paired test of the same block (default 12 pairs, ~72 min, resolves ~1%).\n`;
} else {
body += `> ✅ No significant change.\n`;
}
@@ -841,7 +864,7 @@ jobs:
body += `\n`;
}
- // --- Section 3: Memory growth ---
+ // --- Section 2: Memory growth (only when the growth sweep ran) ---
// Synthetic on purpose: this plots heap against BLOCK SIZE, which needs a
// family of blocks that differ only in transaction count. A real block is
// one point and cannot produce a slope.
diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml
index 5d560a63a..08879ec3f 100644
--- a/.github/workflows/pr_main.yaml
+++ b/.github/workflows/pr_main.yaml
@@ -40,6 +40,14 @@ jobs:
- name: Check ethrex fixture checksums
run: make check-ethrex-fixture-checksums
+ # The benchmark comment harness: verifies every steps.X.outputs.Y consumed in
+ # benchmark-pr.yml has a producer (the orphaned-wiring class that once broke
+ # /bench-growth silently), then renders the full comment scenario suite.
+ # Pure node, no dependencies, ~10 s. Lives in Lint so it runs on every PR —
+ # a path filter would have to enumerate every file that can break the wiring.
+ - name: Check benchmark comment wiring
+ run: node scripts/render_bench_comment.js
+
test-executor:
name: Executor tests
runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index a3471bdb4..25dce43de 100644
--- a/Makefile
+++ b/Makefile
@@ -282,7 +282,8 @@ test-rust: compile-programs-rust
#
# the fixture the rkyv ProgramInput the benchmarks prove (~1 MB)
# the cache the ethrex-replay JSON it was converted from (~2 MB), read only
-# by the converter's tests and by `regen-real-block-fixture`
+# by `regen-real-block-fixture`. The converter's TESTS read a
+# different, upstream-pinned cache — see below.
#
# Fetching a verified binary is the same contract as prepare-sysroot above, and it
# keeps the converter, the ~335-package ethrex host dependency tree and an
@@ -296,7 +297,7 @@ test-rust: compile-programs-rust
# below. It is a regeneration tool for ethrex rev bumps, not a build step.
#
# ---- Repointing to a different block ----
-# These FIVE lines and nothing else. Every path below derives from them, the
+# These SIX lines and nothing else. Every path below derives from them, the
# benchmark scripts and CI resolve the fixture through
# `make -s print-real-block-fixture`, and no workflow, script or env var anywhere
# names a block. Outside this file the repoint touches only REAL_BLOCK_FIXTURE in
@@ -377,9 +378,10 @@ ethrex-real-block-cache:
$(call ensure_verified,$(ETHREX_REAL_BLOCK_CACHE_URL),$(ETHREX_REAL_BLOCK_CACHE_SHA256),$(ETHREX_REAL_BLOCK_CACHE),cache,ETHREX_REAL_BLOCK_CACHE_URL)
# Single source of truth for the benchmark tooling. scripts/bench_verify.sh,
-# scripts/perf_diff.sh and .github/workflows/benchmark-pr.yml read the fixture
-# path from here instead of hardcoding it, so repointing the block above moves
-# every benchmark at once. `-s` on the caller's side keeps the output clean.
+# scripts/bench_abba.sh, scripts/perf_diff.sh and
+# .github/workflows/benchmark-pr.yml read the fixture path from here instead of
+# hardcoding it, so repointing the block above moves every benchmark at once.
+# `-s` on the caller's side keeps the output clean.
print-real-block-fixture:
@echo $(ETHREX_REAL_BLOCK_FIXTURE)
@@ -443,8 +445,11 @@ regen-real-block-fixture: ethrex-real-block-cache
# ethrex host-reference tests live in the detached `tooling/ethrex-tests`
# workspace (ethrex pins rkyv's `unaligned` feature; isolated Cargo.lock).
-# Needs the real-block fixture, so it needs the fixture URL to be set; CI runs the
-# `-offline` variant below in the PR gate and this one only in ethrex-block-converter.yml.
+# Needs the real-block fixture, so it needs the fixture URL to be set. This is a
+# local convenience target: no workflow invokes it. The PR gate spells out the
+# `-offline` variant below inline (pr_main.yaml), and ethrex-block-converter.yml's
+# block-usability job fetches the fixture and runs
+# `test_ethrex_real_block_native` on its own.
test-ethrex: compile-programs-rust ethrex-real-block-fixture
cd tooling/ethrex-tests && cargo test --release -- --include-ignored --skip test_ethrex_real_block_vm
diff --git a/executor/.gitignore b/executor/.gitignore
index 1229bde5c..28b14d098 100644
--- a/executor/.gitignore
+++ b/executor/.gitignore
@@ -1,7 +1,8 @@
/target
/program_artifacts/rust
-# Real-block fixtures (~1 MB): generated by `make ethrex-real-block-fixture`
-# from an ethrex-replay cache, never committed. See tooling/ethrex-block-converter.
+# Real-block fixtures (~1 MB): fetched by `make ethrex-real-block-fixture` from a
+# release asset and verified against a sha256 pinned in the Makefile, never
+# committed. See tooling/ethrex-block-converter.
# One pattern per network the converter accepts (mainnet/hoodi/sepolia — it
# rejects anything else), so repointing ETHREX_REAL_BLOCK_NETWORK in the Makefile
# cannot silently make a ~1 MB fixture committable.
diff --git a/executor/tests/README.md b/executor/tests/README.md
index 80c16d4eb..fdbf47bcb 100644
--- a/executor/tests/README.md
+++ b/executor/tests/README.md
@@ -57,8 +57,11 @@ ethrex_10_transfers.bin
The blocks above are synthetic (N plain ETH transfers over a small genesis).
For a representative workload — real contract execution, real trie depth, real
-bytecode — `make ethrex-real-block-fixture` generates
-`ethrex_hoodi_1265656.bin` from an ethrex-replay cache. It is ~1 MB, so it is
-gitignored and generated on demand rather than committed, and its checksum is
-pinned in `tooling/ethrex-block-converter/README.md` rather than above (the
-checksum script only covers committed fixtures).
+bytecode — `make ethrex-real-block-fixture` downloads
+`ethrex_mainnet_25368371.bin` (1,110,156 B) from the `bench-fixtures-v1` release
+and verifies it against `ETHREX_REAL_BLOCK_FIXTURE_SHA256` in the Makefile before
+moving it into place. It is gitignored rather than committed, so the checksum
+lives next to the URL in the Makefile rather than in the table above (the checksum
+script only covers committed fixtures). See
+`tooling/ethrex-block-converter/README.md` for how the fixture is produced and
+repointed.
diff --git a/scripts/bench_abba.sh b/scripts/bench_abba.sh
index 061c6e407..0baa21db7 100755
--- a/scripts/bench_abba.sh
+++ b/scripts/bench_abba.sh
@@ -37,22 +37,33 @@
# TX_COUNT= ethrex transfer fixture to prove (default 5; use 20 for a
# large continuation trace where GPU-residency wins are visible).
# Ignored when WORKLOAD=real.
-# WORKLOAD=synthetic|real (default synthetic) which block to prove. `real`
+# WORKLOAD=real|synthetic (default real) which block to prove. `real`
# fetches the real-block fixture (block identity lives in the Makefile) and
# forces --continuations; TX_COUNT and CONTINUATIONS do not apply to it.
#
-# On WORKLOAD: the synthetic default is N plain transfers — ecrecover-heavy over a
-# near-empty state — while a real block is keccak- and trie-bound, so a prover change
-# can move the two in opposite directions. `real` is what benchmark-gpu.yml runs by
-# default; `synthetic` stays the default HERE so CPU /bench-abba results remain
-# comparable with everything recorded before.
+# On WORKLOAD: a real block is keccak- and trie-bound, while the synthetic option is
+# N plain transfers — ecrecover-heavy over a near-empty state — so a prover change can
+# move the two in opposite directions. `real` is the default here, in benchmark-gpu.yml
+# and in /bench, so the screen and its tiebreaker resolve the same workload; pass
+# WORKLOAD=synthetic to reproduce a number recorded against that fixture.
#
-# Sizing (ethrex pair-noise sd ~1.2%, 80% power): ~12 pairs for a 1% effect,
-# ~18 for 0.8%, ~32 for 0.6%. Default 20 -> solid on 0.8-1%, ~60% power at 0.6%
-# (if a 20-pair run straddles 0 on a ~0.6%-looking effect, extend to 32).
+# Sizing at WORKLOAD=real, from the paired t-test (resolvable 95% delta =
+# t* x sd / sqrt(N)). The pair-delta sd on the bench runner is NOT yet measured; the
+# two columns bracket it between 1.0% (the GPU box's measured 0.64% plus margin) and
+# 2.0% (sqrt(2) x the runner's measured 1.43% single-run CV):
+#
+# pairs wall resolves (sd 2.0% / sd 1.0%)
+# 8 ~50 min 1.7% / 0.8%
+# 12 ~72 min 1.3% / 0.6% <- workflow default
+# 20 ~1h55m 0.9% / 0.5%
+# 32 ~3h 0.7% / 0.4%
+#
+# Wall assumes epoch 2^22 (158.8 s per prove, two per pair) plus ~8 min of setup.
+# The first real ABBA run MEASURES that sd — read it off the `sd` field of the
+# paired-t line printed below — and this table should be re-pinned to it.
#
# scripts/bench_abba.sh origin/my-pr-branch # vs main, 20 pairs
-# scripts/bench_abba.sh origin/my-pr-branch origin/main 32 # 32 pairs (~0.6%)
+# scripts/bench_abba.sh origin/my-pr-branch origin/main 32 # 32 pairs
set -euo pipefail
@@ -76,7 +87,7 @@ CONTINUATIONS="${CONTINUATIONS:-0}"
EPOCH_SIZE_LOG2="${EPOCH_SIZE_LOG2:-20}"
# ethrex transfer-count fixture to prove (executor/tests/ethrex_${TX_COUNT}_transfers.bin).
TX_COUNT="${TX_COUNT:-5}"
-WORKLOAD="${WORKLOAD:-synthetic}"
+WORKLOAD="${WORKLOAD:-real}"
case "$WORKLOAD" in
synthetic|real) ;;
*) echo "ERROR: WORKLOAD must be 'synthetic' or 'real' (got '$WORKLOAD')." >&2; exit 2 ;;
@@ -136,18 +147,18 @@ if [ ! -f "$ELF_REL" ]; then
export SYSROOT_DIR="${SYSROOT_DIR:-$HOME/.lambda-vm-sysroot}"
make "$ELF_REL"
fi
-if [ ! -f "$INPUT_REL" ]; then
- if [ "$WORKLOAD" = "real" ]; then
- # ~1 MB, gitignored, never in a fresh checkout — and a rented GPU box is always a
- # fresh checkout. Fetched by URL + sha256, not built: no converter, no ethrex host
- # dependency tree, so this costs seconds on the box.
- echo "==> Fetching ethrex real-block fixture (missing)"
- make ethrex-real-block-fixture
- else
- echo "==> Generating ethrex ${TX_COUNT}-transfer fixture (missing)"
- ( cd tooling/ethrex-fixtures && cargo build --release )
- tooling/ethrex-fixtures/target/release/ethrex-fixtures "$TX_COUNT" "$INPUT_REL" distinct
- fi
+if [ "$WORKLOAD" = "real" ]; then
+ # ~1 MB, gitignored, never in a fresh checkout — and a rented GPU box is always a
+ # fresh checkout. Fetched by URL + sha256, not built: no converter, no ethrex host
+ # dependency tree, so this costs seconds on the box. Unconditional on purpose: the
+ # target hashes whatever is on disk on every invocation, which is what catches a
+ # copy left behind by an earlier run in the same rental. A match costs ~35 ms.
+ echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)"
+ make ethrex-real-block-fixture
+elif [ ! -f "$INPUT_REL" ]; then
+ echo "==> Generating ethrex ${TX_COUNT}-transfer fixture (missing)"
+ ( cd tooling/ethrex-fixtures && cargo build --release )
+ tooling/ethrex-fixtures/target/release/ethrex-fixtures "$TX_COUNT" "$INPUT_REL" distinct
fi
ELF="$(cd "$(dirname "$ELF_REL")" && pwd)/$(basename "$ELF_REL")"
INPUT="$(cd "$(dirname "$INPUT_REL")" && pwd)/$(basename "$INPUT_REL")"
diff --git a/scripts/bench_verify.sh b/scripts/bench_verify.sh
index 068b4c864..369041c48 100755
--- a/scripts/bench_verify.sh
+++ b/scripts/bench_verify.sh
@@ -4,12 +4,15 @@
# Reported % = (PR - baseline)/baseline, matching the classic /bench:
# NEGATIVE numbers are improvements (PR faster/smaller); positive = regression.
#
-# TWO arms over the same ethrex 20-tx block, both at blowup=2 / 219 queries:
+# Arms over one block, both at blowup=2 / 219 queries:
# monolithic one VmProof for the whole execution.
# continuations the same block proved as 2^CONT_EPOCH_LOG2-cycle epochs and verified
# as a ContinuationProof bundle — what /bench proves and what an L2
# actually runs, so a verifier change that only moves per-epoch or
# aggregation cost is invisible in the monolithic arm alone.
+# WORKLOAD picks the block: `synthetic` (default) is the ethrex 20-tx one and runs both
+# arms; `real` runs the continuation arm only, since a real block does not fit
+# monolithically. See "Workload" below.
# The continuation arm is best-effort: if its prove or verify fails (it is the
# memory-hungry one) the arm is skipped with a note and the monolithic verdict still
# posts, rather than failing the whole bench.
@@ -27,34 +30,32 @@
# below 6: the exact Wilcoxon's smallest attainable two-sided p is 2/2^n, so at
# n=4 it is 0.125 and the arm can only ever report BORDERLINE, however large and
# clean the effect.
-# WORKLOAD=synthetic|real (default synthetic) which BLOCK both arms prove.
+# WORKLOAD=real|synthetic (default real) which BLOCK both arms prove.
# `real` fetches the real-block fixture (identity lives in the Makefile) and runs
# the continuation arm ONLY — a real block is hundreds of GB monolithically, so
# that arm is skipped rather than left to OOM. See "Workload" below.
-# CONT_EPOCH_LOG2= continuation epoch size (default 20, min 18). For
-# WORKLOAD=real prefer the calibrated tier for the box you are on — 2^22 on the
-# bench runner or a 64 GiB machine, 2^23 on a 128 GiB one (see
-# tooling/ethrex-block-converter/README.md, "Choosing the epoch size"); the default
-# below is chosen for the SYNTHETIC arm. 20 matches
-# scripts/bench_abba.sh, so this arm proves the same bundle shape /bench already
-# proves on the same server — and 20 txs at 2^20 is strictly cheaper than the
-# 100 txs at 2^20 that /bench runs by default, so it can't be the thing that OOMs
-# the box. (`cli prove --epoch-size-log2 --help` measured ethrex 10tx at ~9.5 GB
-# for 2^20 vs ~15.8 GB for 2^21.) Note this does NOT match
-# bench_recursion_cycles.sh's BLOCK_EPOCH_LOG2=21: that arm needs FEW epochs so
-# the bundle fits the guest's 512 MiB private-input cap, a constraint that
-# doesn't apply to host-side verification.
+# CONT_EPOCH_LOG2= continuation epoch size (default 20, min 18). 20 is the
+# laptop-safe setting, not the fast one: prefer the calibrated tier for the box
+# you are on — 2^22 on the bench runner or a 64 GiB machine, 2^23 on a 128 GiB
+# one (see tooling/ethrex-block-converter/README.md, "Choosing the epoch size"),
+# which is what /bench and /bench-abba pin. (`cli prove --epoch-size-log2 --help`
+# measured ethrex 10tx at ~9.5 GB for 2^20 vs ~15.8 GB for 2^21.) Note this does
+# NOT match bench_recursion_cycles.sh's BLOCK_EPOCH_LOG2=21: that arm needs FEW
+# epochs so the bundle fits the guest's 512 MiB private-input cap, a constraint
+# that doesn't apply to host-side verification.
#
# Workload. What this script measures is VERIFY cost, and verify cost is structural in
# the proof — table mix, trace lengths, query count — so the block decides what is being
-# measured, not the loop around it. The synthetic default is 20 plain transfers:
-# ecrecover-heavy over a near-empty state. A real block inverts that mix (keccak- and
-# trie-bound), so a verifier change can move the two differently.
+# measured, not the loop around it. A real block is keccak- and trie-bound; the synthetic
+# option is 20 plain transfers, ecrecover-heavy over a near-empty state, and inverts that
+# mix, so a verifier change can move the two differently.
#
-# `synthetic` stays the default because it is what `/bench-verify` runs and what every
-# number recorded so far used; `real` is the representative one, and costs a ~4.5-4.8
-# min continuation prove per side (cached in $WORK afterwards) before any verify run.
-# Both sides always prove the same block, so a comparison is never mixed.
+# `real` is the default here because it is the representative one and what /bench and
+# /bench-abba measure. It costs a ~2.6 min continuation prove per side (cached in $WORK
+# afterwards) before any verify run, and skips the monolithic arm. `/bench-verify` pins
+# WORKLOAD=synthetic so its two-arm reports stay comparable with the numbers recorded
+# against that fixture. Both sides always prove the same block, so a comparison is
+# never mixed.
set -euo pipefail
@@ -69,7 +70,7 @@ N_PAIRS="${3:-20}"
BENCH_FEATURES="${BENCH_FEATURES:-jemalloc-stats}"
CONT_PAIRS="${CONT_PAIRS:-8}"
CONT_EPOCH_LOG2="${CONT_EPOCH_LOG2:-20}"
-WORKLOAD="${WORKLOAD:-synthetic}"
+WORKLOAD="${WORKLOAD:-real}"
case "$WORKLOAD" in
synthetic|real) ;;
*) echo "ERROR: WORKLOAD must be 'synthetic' or 'real' (got '$WORKLOAD')." >&2; exit 2 ;;
@@ -179,17 +180,18 @@ if [ ! -f "$ELF_REL" ]; then
export SYSROOT_DIR="${SYSROOT_DIR:-$HOME/.lambda-vm-sysroot}"
make "$ELF_REL"
fi
-if [ ! -f "$INPUT_REL" ]; then
- if [ "$WORKLOAD" = "real" ]; then
- # ~1 MB, gitignored, never in a fresh checkout. Fetched by URL + sha256, not built:
- # no converter and no ethrex host dependency tree on this path.
- echo "==> Fetching ethrex real-block fixture (missing)"
- make ethrex-real-block-fixture
- else
- echo "==> Generating ethrex 20-transfer fixture (missing)"
- ( cd tooling/ethrex-fixtures && cargo build --release )
- tooling/ethrex-fixtures/target/release/ethrex-fixtures 20 "$INPUT_REL" distinct
- fi
+if [ "$WORKLOAD" = "real" ]; then
+ # ~1 MB, gitignored, never in a fresh checkout. Fetched by URL + sha256, not built:
+ # no converter and no ethrex host dependency tree on this path. Unconditional on
+ # purpose: the target hashes whatever is already on disk on every invocation, which
+ # is how a stale copy left by an earlier block or an interrupted fetch gets caught.
+ # A match costs ~35 ms, so there is nothing to gate it on.
+ echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)"
+ make ethrex-real-block-fixture
+elif [ ! -f "$INPUT_REL" ]; then
+ echo "==> Generating ethrex 20-transfer fixture (missing)"
+ ( cd tooling/ethrex-fixtures && cargo build --release )
+ tooling/ethrex-fixtures/target/release/ethrex-fixtures 20 "$INPUT_REL" distinct
fi
ELF="$(cd "$(dirname "$ELF_REL")" && pwd)/$(basename "$ELF_REL")"
INPUT="$(cd "$(dirname "$INPUT_REL")" && pwd)/$(basename "$INPUT_REL")"
diff --git a/scripts/perf_diff.sh b/scripts/perf_diff.sh
index a6c0380be..2c359a548 100755
--- a/scripts/perf_diff.sh
+++ b/scripts/perf_diff.sh
@@ -13,22 +13,23 @@
#
# USAGE (on the bench server):
# scripts/perf_diff.sh REF_A [REF_B=origin/main]
-# Env: WORKLOAD=synthetic|real (default synthetic) picks the block to profile;
+# Env: WORKLOAD=real|synthetic (default real) picks the block to profile;
# EPOCH_SIZE_LOG2= (default 22) sizes the epoch, WORKLOAD=real only.
# 22 is the calibrated bench-runner tier, matching /bench; use 23 on a
# 128 GiB box (tooling/ethrex-block-converter/README.md, "Choosing the epoch size").
#
# Pick the workload that matches the run you are localizing, because the symbol
-# mix follows the block: the synthetic default is 20 plain transfers (8.73M cycles,
-# 411 keccak calls, 80 ecsm calls) and a real block inverts that (50.78M cycles,
-# 10,478 keccak, 116 ecsm), so a hot symbol in one need not be hot in the other.
+# mix follows the block: the real default is 50.78M cycles, 10,478 keccak calls and
+# 116 ecsm calls, and the synthetic option (20 plain transfers) inverts that at
+# 8.73M cycles, 411 keccak, 80 ecsm — so a hot symbol in one need not be hot in the other.
# Both counts are from the same guest ELF (merge fdb92f67, main @ 9ccdaf2, clang 21);
# they move with guest optimisation (#861's thin LTO) and ~2% with the clang major, so
# pin the ELF when quoting one.
#
# WORKLOAD=real also switches to a continuation prove (monolithic would need ~240 GB
-# at that trace length), which is ~4-5 min per recording — five recordings, so budget
-# ~25 min, plus ~1.2 GB of disk per bundle and ~32 GiB of RAM at the default epoch.
+# at that trace length), which is 158.8 s per recording on the bench runner — five
+# recordings, so budget ~13 min of proving, plus ~1.2 GB of disk per bundle and ~52 GB
+# of RAM at the default epoch.
#
# Produces:
# - two perf-diff tables (recorded twice per side, interleaved B A B A —
@@ -46,10 +47,10 @@ if [ $# -lt 1 ]; then
fi
REF_A="$1"
REF_B="${2:-origin/main}"
-WORKLOAD="${WORKLOAD:-synthetic}"
+WORKLOAD="${WORKLOAD:-real}"
# 2^22: the calibrated tier for the bench server this script targets, same as
-# /bench's real-block arm. Memory picks it, not speed — 32.2 GiB fits a >=64 GiB box with ~50%
-# headroom where 2^23's 60 GiB does not.
+# /bench's real-block arm. Memory picks it, not speed — that server peaks at ~52 GB on
+# a >=64 GiB floor, and 2^23 measured 60 GiB on a roomier box, so it would not fit here.
EPOCH_SIZE_LOG2="${EPOCH_SIZE_LOG2:-22}"
case "$WORKLOAD" in
synthetic|real) ;;
@@ -76,16 +77,16 @@ fi
command -v perf >/dev/null 2>&1 || { echo "ERROR: perf not installed (linux-tools)." >&2; exit 1; }
[ -f "$ELF_REL" ] || { echo "ERROR: missing $ELF_REL — run bench_abba.sh once (it builds the guest)." >&2; exit 1; }
-if [ ! -f "$INPUT_REL" ]; then
- if [ "$WORKLOAD" = "real" ]; then
- # ~1 MB, gitignored, never in a fresh checkout; fetch rather than abort. This is
- # a URL + sha256 download, not a build.
- echo "==> Fetching ethrex real-block fixture (missing)"
- make ethrex-real-block-fixture
- else
- echo "ERROR: missing $INPUT_REL — run bench_abba.sh once (it builds the fixture)." >&2
- exit 1
- fi
+if [ "$WORKLOAD" = "real" ]; then
+ # ~1 MB, gitignored, never in a fresh checkout; fetch rather than abort. This is a
+ # URL + sha256 download, not a build. Unconditional on purpose: the target hashes
+ # whatever is on disk on every invocation, which is how a stale copy gets caught.
+ # A match costs ~35 ms.
+ echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)"
+ make ethrex-real-block-fixture
+elif [ ! -f "$INPUT_REL" ]; then
+ echo "ERROR: missing $INPUT_REL — run WORKLOAD=synthetic scripts/bench_abba.sh once (it generates the synthetic fixture; the default workload is real and would not)." >&2
+ exit 1
fi
echo "==> Workload: $WORKLOAD ($INPUT_REL${CONT_ARGS:+, continuations epoch 2^$EPOCH_SIZE_LOG2})"
diff --git a/scripts/render_bench_comment.js b/scripts/render_bench_comment.js
index eee39f231..abc994f4c 100644
--- a/scripts/render_bench_comment.js
+++ b/scripts/render_bench_comment.js
@@ -4,7 +4,7 @@
//
// WHY: that comment is ~200 lines of JS embedded in YAML, and the only other way to
// see its output is to push and run a real benchmark on the shared bench server
-// (~15-20 min of occupancy). This extracts the `script:` block from the workflow,
+// (~15 min of occupancy). This extracts the `script:` block from the workflow,
// runs it against synthetic env values, and prints the markdown — so a wording or
// formatting change is checked in a second.
//
@@ -12,6 +12,11 @@
// that have actually broken before: sign handling on the verdict lines, the
// no-baseline and block-mismatch paths, and the fixture-unavailable footer.
//
+// The one thing it does check is step wiring, because the scenarios cannot: they
+// hand env to the script directly, so a `steps.X.outputs.Y` that no step writes
+// still renders perfectly here while arriving empty in CI — which drops a whole
+// section with no error anywhere. That check runs first and exits non-zero.
+//
// USAGE: node scripts/render_bench_comment.js
//
// Add a scenario by adding a key to `scenarios`; each is the env the comment step
@@ -33,6 +38,47 @@ for (let i = start + 1; i < lines.length; i++) {
}
const src = body.join('\n');
+// --- Wiring check: every consumed step output has a producer --------------------
+// Steps are ` - name:` at six spaces; a step's outputs are the `key=` it echoes
+// into $GITHUB_OUTPUT. A step that builds key names at runtime (`echo "$key=..."`)
+// cannot be enumerated statically, so it is skipped rather than guessed at.
+const stepStart = (l) => /^ {6}- name:/.test(l);
+const produced = new Map(); // id -> Set of statically enumerable output keys
+const allIds = new Set(); // every step id, so a typoed reference cannot pass
+const opaque = new Set(); // outputs not statically enumerable: `uses:` actions
+ // (native outputs) and runtime-built `echo "$key=..."`
+for (let i = 0; i < lines.length; i++) {
+ if (!stepStart(lines[i])) continue;
+ let end = i + 1;
+ while (end < lines.length && !stepStart(lines[end])) end++;
+ const block = lines.slice(i, end).join('\n');
+ const id = (block.match(/^ {8}id: (\S+)$/m) || [])[1];
+ if (!id) continue;
+ allIds.add(id);
+ if (/^ {8}uses:/m.test(block) || /echo "\$\{?[A-Za-z_][^"]*=/.test(block)) {
+ opaque.add(id);
+ continue;
+ }
+ // A run: step with no $GITHUB_OUTPUT produces nothing — reads from it must fail.
+ produced.set(id, new Set(
+ block.includes('$GITHUB_OUTPUT')
+ ? [...block.matchAll(/echo "([a-z0-9_]+)=/g)].map(m => m[1])
+ : []
+ ));
+}
+const problems = [];
+for (const m of wf.matchAll(/steps\.([a-z0-9_-]+)\.outputs\.([a-z0-9_]+)/g)) {
+ const [ref, id, key] = m;
+ if (!allIds.has(id)) problems.push(`${ref} (no step with id '${id}' — typo?)`);
+ else if (opaque.has(id)) continue; // uses:/dynamic — cannot verify statically
+ else if (!produced.get(id).has(key)) problems.push(`${ref} (step '${id}' never writes '${key}')`);
+}
+if (problems.length) {
+ console.error(`Unwired step outputs:\n ${[...new Set(problems)].join('\n ')}`);
+ process.exit(1);
+}
+console.log(`Wiring OK: ${produced.size} producer steps verified, ${opaque.size} opaque (uses:/dynamic) skipped, no unknown step ids.`);
+
const captured = [];
const github = { rest: { issues: {
listComments: async () => ({ data: [] }),
@@ -57,8 +103,11 @@ const scenarios = {
REAL_TIME_DIFF: '-58.600', REAL_TIME_PCT: '-17.2', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0',
},
'C: clear regression': {
+ // Diffs are unsigned as the Compare step prints them (awk "%.3f"/"%.1f", `$((a-b))`);
+ // the renderer's own `fmt` adds the sign, so feeding one signed here would test
+ // a format CI never produces.
...REAL, BASE_REAL_TIME: '240.000', BASE_REAL_PEAK: '32975',
- REAL_TIME_DIFF: '+41.400', REAL_TIME_PCT: '17.3', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0',
+ REAL_TIME_DIFF: '41.400', REAL_TIME_PCT: '17.3', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0',
},
'D: unresolved middle band (3-10%)': {
...REAL, BASE_REAL_TIME: '300.000', BASE_REAL_PEAK: '32975',
@@ -75,7 +124,9 @@ const scenarios = {
REAL_MISMATCH: 'ethrex_hoodi_1265656.bin',
},
'H: fixture unavailable': { COMMIT_SHA: 'c4e42b2900', REAL_EPOCH_LOG2: '22', BASELINE_SRC: 'cached' },
- 'I: with growth sweep (push-style)': {
+ // The growth sweep reaches a comment only on /bench-growth: run_growth is also true
+ // on push and workflow_dispatch, but the Comment step is skipped for both.
+ 'I: /bench-growth — real block plus the growth sweep': {
...REAL, BASE_REAL_TIME: '288.900', BASE_REAL_PEAK: '32975',
REAL_TIME_DIFF: '-7.500', REAL_TIME_PCT: '-2.6', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0',
PR_GROWTH_HEAPS: '18756/26966/34748/43896/50431',
diff --git a/tooling/ethrex-block-converter/.gitignore b/tooling/ethrex-block-converter/.gitignore
index 6d50c2d01..d4c9a8fb7 100644
--- a/tooling/ethrex-block-converter/.gitignore
+++ b/tooling/ethrex-block-converter/.gitignore
@@ -1,4 +1,6 @@
/target
-# Downloaded ethrex-replay cache JSONs (~1.5 MB each), fetched on demand by
-# `make ethrex-real-block-fixture`.
+# Downloaded ethrex-replay cache JSONs (~1.5 MB each), plus the rev stamp that
+# invalidates them. Fetched on demand by `make ethrex-real-block-cache` (the
+# benchmark block, for `regen-real-block-fixture`) and by
+# `make test-ethrex-real-block-converter` (this crate's own pinned test block).
/caches/
diff --git a/tooling/ethrex-block-converter/Cargo.toml b/tooling/ethrex-block-converter/Cargo.toml
index b1796fa60..f61e68a68 100644
--- a/tooling/ethrex-block-converter/Cargo.toml
+++ b/tooling/ethrex-block-converter/Cargo.toml
@@ -35,11 +35,12 @@ serde_json = "1"
# Not used by this crate's code directly — declared to pin k256's feature set.
# `lambda-vm-ethrex-crypto` (a dev-dependency) is otherwise the only thing asking
# for `expose-field`, and under resolver 3 dev-dependency features are not unified
-# into builds that exclude dev-dependencies: `cargo run` (the
-# `make ethrex-real-block-fixture` step) and the `cargo test` that follows it in
-# CI would resolve two different k256 feature sets, recompiling k256 and every
-# ethrex crate above it in the same job. Declaring the features here puts them in
-# both resolutions. Keep in sync with crypto/ethrex-crypto's k256 line; verify:
+# into builds that exclude dev-dependencies. This crate is built both ways against
+# one target dir: `cargo run` for `make regen-real-block-fixture`, `cargo test` for
+# `make test-ethrex-real-block-converter`. Without this line the two resolve
+# different k256 feature sets, so alternating between them recompiles k256 and every
+# ethrex crate above it. Declaring the features here puts them in both resolutions.
+# Keep in sync with crypto/ethrex-crypto's k256 line; verify:
# cargo tree -e features,no-dev -i k256 # must list expose-field
# cargo tree -e features -i k256 # must match the above
k256 = { version = "=0.13.4", default-features = false, features = ["arithmetic", "expose-field"] }
diff --git a/tooling/ethrex-block-converter/README.md b/tooling/ethrex-block-converter/README.md
index 78dc4fd2e..17cf8a2f8 100644
--- a/tooling/ethrex-block-converter/README.md
+++ b/tooling/ethrex-block-converter/README.md
@@ -20,14 +20,21 @@ All are measured.
| serialized size | 32,766 B | 1,110,156 B |
| cycles | 8,734,622 | **50,781,557** |
| keccak / ecsm calls | 411 / 80 | **10,478** / 116 |
-| keccaks per ecrecover | 5.1 | **90** |
+| keccaks per ECSM call | 5.1 | **90** |
+
+The last row counts keccaks per **ECSM call**, which is the ratio of the two rows
+above it. One ecrecover issues four ECSM ecalls — `lincomb2_with_oracle` in
+`crypto/ethrex-crypto/src/lib.rs` makes four oracle queries, each an `ecsm_mul` — so
+per *ecrecover* the figures are 20.6 and 361, and the 80 and 116 above are 4x the 20
+synthetic and 29 real signature checks.
Note the real block uses ~5.8x the gas and costs ~5.8x the cycles here, and that it
inverts the crypto mix: the synthetic block is ecrecover-bound, the real one keccak-
and trie-bound. That inversion is the point — a prover change can move the two
numbers in opposite directions. (The gas and cycle ratios agreeing is a coincidence
-of this pair, not a rule: cycles/gas is 20.9 here and ranges ~21–38 across the
-candidate blocks.)
+of this pair, not a rule: cycles/gas is 20.9 for this block on this ELF, and spans
+29.7–38.2 across the three candidate blocks when all three are measured on one
+common pre-LTO ELF — a ~29% spread.)
**Pin the ELF whenever you quote a cycle count.** Counts above were measured on this
branch at merge `fdb92f67` (main @ `9ccdaf2`), guest built with **clang 21.1.8**.
@@ -140,28 +147,47 @@ it — see [Measured cost of candidate blocks](#measured-cost-of-candidate-block
| Where | How to run it | Cost (current default) |
|---|---|---|
-| `benchmark-pr.yml` | **`/bench`** on a PR — the only workload it proves; also push to main and `workflow_dispatch` | 3 runs, ~4–5 min each (first run at epoch 2^22 confirms) |
-| `scripts/bench_verify.sh` | `WORKLOAD=real scripts/bench_verify.sh [` | ~4.5–4.8 min per side, then cached |
-| `scripts/perf_diff.sh` | `WORKLOAD=real scripts/perf_diff.sh ][` | 5 recordings, so ~25 min |
-| `benchmark-gpu.yml` | `/bench-gpu` on a PR — **the default there** | 59.87 s/prove on an RTX 5090 (see below) |
-| `scripts/bench_abba.sh` | `WORKLOAD=real scripts/bench_abba.sh ][` | ~4 h at 20 pairs — **manual only** |
+| `benchmark-pr.yml` | **`/bench`** on a PR — also push to main and `workflow_dispatch` | 3 runs, 158.8 s each (~8 min of proving) |
+| `bench-abba.yml` | **`/bench-abba [N]`** on a PR | ~72 min at the default 12 pairs (see below) |
+| `benchmark-gpu.yml` | **`/bench-gpu [N]`** on a PR | 59.87 s/prove on an RTX 5090 (see below) |
+| `scripts/bench_verify.sh` | `scripts/bench_verify.sh ][` | ~2.6 min per side, then cached |
+| `scripts/perf_diff.sh` | `scripts/perf_diff.sh ][` | 5 recordings, so ~13 min of proving |
+| `scripts/bench_abba.sh` | `scripts/bench_abba.sh ][ [base] [pairs]` | 2 x 158.8 s per pair |
+
+This block is what all three scripts prove by default (`WORKLOAD=real`); pass
+`WORKLOAD=synthetic` for the N-plain-transfer fixture instead. `/bench-verify` is the
+one flow that pins `synthetic`, because it reports a monolithic arm as well as a
+continuation one and a real block does not fit monolithically.
None of them hardcode the fixture path or a block number — they read the path from
-`make -s print-real-block-fixture` and run `make ethrex-real-block-fixture` when
-the `.bin` is absent.
-
-**`/bench-abba` is deliberately NOT wired to the real block.** The script supports
-it (`WORKLOAD=real`, which is what the GPU workflow drives), but the CPU ABBA
-workflow still defaults to the synthetic fixture: 20 pairs × 2 proves × ~4.5–4.8 min
-is **~3 hours** on the one shared bench runner, which every other bench queues behind.
-Run it by hand when a paired test on a real workload is worth that; the option is
-here rather than a footgun on a comment trigger.
-
-**Both bench flows prove this block.** `/bench` runs it sampled on the shared
-runner, against the cached baseline main publishes; `/bench-gpu [N]` runs it as
-N A/B/B/A pairs on a rented box, comparing PR vs main on the same machine —
-absolute GPU times are host-CPU-dependent, so only same-box deltas are
-meaningful.
+`make -s print-real-block-fixture` and run `make ethrex-real-block-fixture` on every
+invocation, so the digest is re-checked rather than trusted.
+
+**Every bench flow proves this block.** `/bench` runs it sampled on the shared
+runner, against the cached baseline main publishes; `/bench-abba [N]` runs it as
+N A/B/B/A pairs on that same runner; `/bench-gpu [N]` runs the same pairs on a
+rented box, comparing PR vs main on the same machine — absolute GPU times are
+host-CPU-dependent, so only same-box deltas are meaningful.
+
+**Escalating from `/bench` to `/bench-abba`.** `/bench` resolves about 3%: three
+runs of a 158.8 s prove, so it reports 3–10% as unresolved rather than as a
+verdict. The paired test resolves a 95% delta of `t* × sd / sqrt(N)`, where `sd`
+is the pair-delta standard deviation on the runner. **That sd is not measured
+yet.** The columns below bracket it between 1.0% — the GPU box's measured 0.64%
+pair sd plus margin — and 2.0%, which is `sqrt(2) ×` this runner's measured 1.43%
+single-run CV:
+
+| pairs | wall | resolves (sd 2.0%) | resolves (sd 1.0%) |
+|---|---|---|---|
+| 8 | ~50 min | 1.7% | 0.8% |
+| **12** | **~72 min** | **1.3%** | **0.6%** |
+| 20 | ~1h55m | 0.9% | 0.5% |
+| 32 | ~3h | 0.7% | 0.4% |
+
+12 pairs is the default. Wall is two 158.8 s proves per pair plus ~8 min of
+setup. The first real `/bench-abba` run **measures** the sd — it is the `sd`
+field of the paired-t line in the result comment — and this table should be
+re-pinned to that value once it exists.
**GPU baseline (measured), and why the GPU epoch is 2^22.** On an RTX 5090 (32,607 MiB)
against main @ `9ccdaf2`, same fixture and CLI, one prove per setting:
@@ -179,31 +205,36 @@ this one. `benchmark-gpu.yml` defaults the real-block path to 2^22 for this reas
raw traces are in `~/workspace/lambda_vm_bench_cache/gpu_epoch_calib_2026-07-31/`
(`PROVENANCE.txt`).
-That default is **GPU-path only**. `bench_abba.sh` still defaults to 2^20 for the CPU
-`/bench-abba`, and the CLI's own `DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2` is still 20:
-2^22 needs ~28 GiB of *host* memory on a CPU build, which would break laptops. VRAM
-binds on one path and host RAM on the other, so they are not the same question and the
-GPU number must not be copied across. **The CPU-server epoch recommendation is pending
-calibration.**
+2^22 is also what the CPU runner uses, but the two arrive there for different reasons
+and must not be derived from each other: VRAM binds on the GPU path and host RAM on the
+CPU one. The workflows pin it on both sides (`GPU_REAL_EPOCH_LOG2` here,
+`REAL_BLOCK_EPOCH_LOG2` in `benchmark-pr.yml`, `ABBA_REAL_EPOCH_LOG2` in
+`bench-abba.yml`); `bench_abba.sh` and `bench_verify.sh` still *default* to 2^20, as
+does the CLI's `DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2`, because 2^22 needs ~32 GiB of
+host memory on a CPU build (peak RSS on the calibration box) and would break laptops.
+See [Choosing the epoch size](#choosing-the-epoch-size) for the CPU tiers.
-The CPU bench runner is roughly **4.5–4.8x** the GPU wall time for the same block.
+The CPU bench runner is roughly **2.65x** the GPU wall time for the same block: 158.8 s
+median against the calibration RTX 5090's 59.87 s, both at epoch 2^22.
-Do not derive one from the other in general: the CPU rate (5.31–5.62 s/Mcycle) does not
-transfer to the GPU, and the RTX 5090 sweep found the prover CPU-bound at the serial
-producer above epoch 2^21, so GPU time lands closer to CPU time than a naive
-device-throughput estimate suggests.
+Do not derive one from the other in general: the CPU rate (3.13 s/Mcycle on this
+block) does not transfer to the GPU, and the RTX 5090 sweep found the prover
+CPU-bound at the serial producer above epoch 2^21, so GPU time lands closer to CPU
+time than a naive device-throughput estimate suggests.
**Continuations are mandatory, not a tuning choice.** Peak heap on a monolithic
prove grows ~4.9 GB per million cycles on this workload family (measured on the
bench server: `10,728 MB + 2,007 MB/transfer`, R² = 0.998 across 4→20 transfers),
so this block would need **~240 GB** monolithically — and a heavier candidate far
more. `--continuations` makes peak heap a function of the epoch size instead of the
-trace length, so the same block fits in **~32 GiB** at epoch 2^22, the setting both the
-CPU bench runner and the GPU path now use — picked by host RAM on one and by VRAM on
-the other. See [Choosing the epoch size](#choosing-the-epoch-size) for the full curve
-and the other tiers. The
-bundle on disk is ~1.9 GB; note that a heavier block pushes it past 2 GiB, which
-needs rkyv `pointer_width_64` to serialize.
+trace length, so the same block fits in **~32 GiB** at epoch 2^22 on the calibration
+box — the setting both the CPU bench runner and the GPU path now use, picked by host
+RAM on one and by VRAM on the other. Host peaks are machine-specific: the bench runner
+itself measured **~52 GB** of peak heap for the same block and epoch. See
+[Choosing the epoch size](#choosing-the-epoch-size) for the full curve and the other
+tiers. The bundle on disk is ~1.15 GB (1.12 GB on the GPU path); a block would have to
+be ~1.9x heavier to push it past the 2 GiB (2.147 GB) rkyv offset limit, which needs
+`pointer_width_64` to serialize.
**`/bench` proves this block and nothing else** — 3 sampled runs against the cached
3-run baseline main publishes on every push. `/bench N` changes the sample count
@@ -216,20 +247,22 @@ make twice:
1. Its only unique coverage was the **monolithic** prove path, which is vestigial —
reportedly slower than a single-epoch continuation. Spending runner time to cover
a path we intend to delete is not a trade worth making.
-2. Its crypto mix is one **no real block has**: 8.83 ECSM per Mcycle, against a real
- block's ~1.3–1.6. Screening against it tunes the prover for a worst case that
+2. Its crypto mix is one **no real block has**: 9.16 ECSM per Mcycle, against a real
+ block's 2.28. Screening against it tunes the prover for a worst case that
cannot occur.
The synthetic fixtures themselves are not gone — `/bench-growth` still sweeps them for
a heap-vs-block-size slope, which needs a family of blocks and so cannot come from one
-real one. `/bench-abba` and `/bench-verify` are untouched.
+real one, and `/bench-verify` still proves the 20-transfer block so it can report a
+monolithic arm as well as a continuation one.
**The cost is a shared resource.** One runner carries every `/bench`, `/bench-abba`
-and `/bench-verify` in the repo, and a `/bench` now occupies it for **~15–20 min**, on
-every comment and every push to main. `BENCH_RUNS_REAL` in `benchmark-pr.yml` is the
-dial. Its per-run time on that runner is still unmeasured — the epoch calibration ran
-on a different box — so the first run is what tells you whether 3 was the right
-number; above ~6 min a run, turn it down.
+and `/bench-verify` in the repo, and a `/bench` now occupies it for **~15 min**, on
+every comment and every push to main — ~8 min of that is the three proves, the rest is
+checkout, a two-sided build, the fixture fetch and the guest ELF. `BENCH_RUNS_REAL` in
+`benchmark-pr.yml` is the dial. A run measured **158.8 s** on that runner (median of 3,
+2.8% spread, 13 epochs, epoch 2^22), so 3 runs is the right count; the dial is there
+for a future block or prover change that takes a run past ~6 min.
Cycle counts here (8.73M synthetic, 50.78M real) are from merge `fdb92f67` (main @
`9ccdaf2`, clang 21). They move with guest optimisation and ~2% with the clang major,
@@ -313,13 +346,13 @@ repointed ~1 MB fixture cannot become committable by accident.
Cost is a property of the block, so it changes with the repoint. All figures are
measured, never derived from gas — **cycles per gas is not constant** (20.9 for the
-current default), so ranking candidates by gas mispredicts cost.
+current default), so sizing a candidate from its gas mispredicts cost.
**Current default — main-vintage (merge `fdb92f67`, main @ `9ccdaf2`):**
| block | gas | cycles | GPU prove (RTX 5090) | CPU prove | proof | fixture |
|---|---|---|---|---|---|---|
-| **mainnet 25368371** | 2.43M | **50,781,557** (clang 21)]
50,713,534 (clang 18) | **59.87 s** @ epoch 2^22 | ~4.5–4.8x the GPU wall | ~1.9 GB | 1,110,156 B, `61eba49b…` |
+| **mainnet 25368371** | 2.43M | **50,781,557** (clang 21)
50,713,534 (clang 18) | **59.87 s** @ epoch 2^22 | **158.8 s** @ epoch 2^22 (2.65x the GPU wall) | 1.15 GB CPU / 1.12 GB GPU | 1,110,156 B, `61eba49b…` |
Epoch 2^22 is the GPU recommendation: VRAM binds, 2^22 leaves 28.9% headroom on a
32 GiB card and 2^23 does not fit one at all. See
@@ -340,10 +373,13 @@ All three clear the usability screen. Add a row rather than editing the wiring,
say which ELF a number came from.
Two things these numbers show that a gas-based estimate would have got wrong, and both
-survive the vintage change because they are same-ELF comparisons. **Gas does not order
-cost:** 25453112 carries *more* gas than hoodi 1265656 yet costs ~25% fewer cycles.
-And **fixture size does not track cost** either: the current default is the cheapest
-block and the middle-sized fixture.
+survive the vintage change because they are same-ELF comparisons. **Gas does not size
+cost:** on one common pre-LTO ELF the three blocks run at 30.8, 29.7
+and 38.2 cycles per gas, so budgeting a candidate from gas alone is off by up to ~29%
+— 25453112 and hoodi 1265656 sit within 4% of each other on gas (4.24M vs 4.40M) yet
+25453112 costs ~25% fewer cycles (125.9M vs 168.3M). Gas happens to *order* these
+three correctly; it does not size them. And **fixture size does not track cost**
+either: the current default is the cheapest block and the middle-sized fixture.
The default is the cheapest of the three, which matters because the CPU workload sits
on a single shared bench runner. It was also the only block in a 90-day Dune sweep
@@ -359,7 +395,7 @@ machine, not of the block**. Three tiers, all measured:
| where | epoch | why |
|---|---|---|
| GPU, 32 GiB card | **2^22** | VRAM-bound — 2^23 does not fit |
-| CPU bench runner (≥64 GiB) | **2^22** | host-RAM-bound — 2^23's 60 GiB does not fit safely |
+| CPU bench runner (≥64 GiB) | **2^22** | host-RAM-bound — it already peaks at ~52 GB here, and 2^23 measured 60 GiB on a roomier box |
| CPU server, 128 GiB class | **2^23** | the knee; 2^24 fits but is not worth it |
| laptops (CLI default) | **2^20** | unchanged, so a plain `cli prove` still works |
@@ -385,10 +421,12 @@ A main-vintage anchor also ran on the same box: 2^23 = 344.12 s / 58.87 GiB, i.e
faster than branch vintage at the same memory — as expected, since #861 cut cycles and
peak RSS is set by the epoch size rather than the trace length.
-**Do not read absolute seconds off this table for another machine.** The calibration box
-runs ~8.6 s/Mcycle against the bench runner's 5.31–5.62, so the *ratios* between epochs
-transfer and the wall times do not. `/bench`'s real-block arm now runs at 2^22 (it was
-2^21); the first run after that change establishes the runner's real number.
+**Do not read absolute seconds or peak memory off this table for another machine.** This
+box took 397.88 s at 2^22 on its branch-vintage ELF; the bench runner takes 158.8 s at
+the same epoch on the main-vintage one — a 2.5x gap, against the ~6% the two cycle
+counts differ by. The *ratios* between epochs transfer; the wall times do not. Memory
+transfers no better: the bench runner peaks at ~52 GB of heap at 2^22 where this box
+measured 32.21 GiB RSS.
### Verifying a repoint
diff --git a/tooling/ethrex-tests/tests/ethrex.rs b/tooling/ethrex-tests/tests/ethrex.rs
index eaa811bf4..6374f2f43 100644
--- a/tooling/ethrex-tests/tests/ethrex.rs
+++ b/tooling/ethrex-tests/tests/ethrex.rs
@@ -89,14 +89,15 @@ fn test_ethrex_empty_block() {
const REAL_BLOCK_FIXTURE: &str = "ethrex_mainnet_25368371.bin";
-/// Host-only acceptance gate for the real-block fixture produced by
-/// `tooling/ethrex-block-converter` (`make ethrex-real-block-fixture`): the block is
+/// Host-only acceptance gate for the real-block fixture that
+/// `make ethrex-real-block-fixture` fetches and checksums (produced offline by
+/// `tooling/ethrex-block-converter`): the block is
/// re-executed statelessly against its own witness, so a successful run means
/// the recovered tries, codes and headers reproduce the header's post-state
/// root. Needs no guest ELF, which is what keeps it runnable where the RV64
/// toolchain isn't available.
///
-/// Checks the *serialized artifact* specifically — that the committed rkyv
+/// Checks the *serialized artifact* specifically — that the published rkyv
/// bytes deserialize and execute — which is why it reads the `.bin` rather
/// than converting the cache itself.
///