diff --git a/.github/workflows/bench-abba.yml b/.github/workflows/bench-abba.yml index aa066bbd5..23e323d57 100644 --- a/.github/workflows/bench-abba.yml +++ b/.github/workflows/bench-abba.yml @@ -1,18 +1,37 @@ name: Bench ABBA tiebreaker -# Drift-free paired (A/B/B/A) prover benchmark for resolving small (~1%) deltas the -# cheap PR benchmark can't confirm. At the default workload it OCCUPIES THE SINGLE -# BENCH SERVER FOR SEVERAL HOURS, so it NEVER auto-triggers -- it runs only on an -# explicit `/bench-abba` comment on a PR. +# Drift-free paired (A/B/B/A) prover benchmark for resolving the small deltas /bench +# cannot confirm. It proves the SAME real block /bench does, so a 3-10% reading there +# escalates to a paired test of the same workload rather than a different one. It +# OCCUPIES THE SINGLE BENCH SERVER FOR OVER AN HOUR, so it NEVER auto-triggers -- it +# runs only on an explicit `/bench-abba` comment on a PR. # -# Syntax: "/bench-abba [N] [cont[TX]|mono[TX]]" (default: 20 pairs, ethrex 100tx -# --continuations; "cont10" is the quick coarse option, "mono[TX]" the legacy -# monolithic prove). Cont proofs need rkyv pointer_width_64 on both sides, so PR -# branches older than that fix must rebase before a cont bench. +# Syntax: "/bench-abba [N]", where N is the pair count (default 12). +# +# Resolution, from the paired t-test (resolvable 95% delta = t* x sd / sqrt(N)). The +# pair-delta sd on this runner is NOT yet measured; the two columns bracket it between +# 1.0% (the GPU box's measured 0.64% plus margin) and 2.0% (sqrt(2) x this runner's +# measured 1.43% single-run CV): +# +# pairs wall resolves (sd 2.0% / sd 1.0%) +# 8 ~50 min 1.7% / 0.8% +# 12 ~72 min 1.3% / 0.6% <- default +# 20 ~1h55m 0.9% / 0.5% +# 32 ~3h 0.7% / 0.4% +# +# Wall is 2 x 158.8 s per pair (the runner's measured prove at epoch 2^22) plus ~8 min +# of setup. The FIRST run measures the sd — it is the `sd` field of the paired-t line +# in the result comment — and this table should be re-pinned to it then. on: issue_comment: types: [created] +env: + # The epoch /bench proves this block at, so the tiebreaker resolves the same + # configuration and not just the same block. Memory picks it: this runner peaks at + # ~52 GB here against its >=64 GiB floor. See benchmark-pr.yml's REAL_BLOCK_EPOCH_LOG2. + ABBA_REAL_EPOCH_LOG2: "22" + concurrency: # See bench-verify.yml. The single self-hosted bench runner serializes across PRs. group: ${{ startsWith(github.event.comment.body, '/bench-abba') && format('bench-abba-{0}', github.event.issue.number) || format('bench-abba-ignore-{0}', github.run_id) }} @@ -31,9 +50,10 @@ jobs: startsWith(github.event.comment.body, '/bench-abba') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: [self-hosted, bench] - # Hang guardrail, not expected duration: a cont100 CPU prove is ~7 min, so the - # default 20 pairs runs ~4.5-5 hr and the 40-pair clamp ~9.5 hr, plus builds. - timeout-minutes: 720 + # Hang guardrail, not expected duration: a real-block pair is 2 x 158.8 s = ~5.3 min, + # so the default 12 pairs runs ~72 min and the 40-pair clamp ~3.7 hr, plus up to + # ~30 min of two-sided build on a cold cache. + timeout-minutes: 360 steps: - name: Resolve PR head + bench config id: cfg @@ -50,76 +70,38 @@ jobs: # force-push race mid-run. HEAD_SHA=$(gh pr view "$PR_NUM" --repo "$GITHUB_REPOSITORY" --json headRefOid -q .headRefOid) echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" - # Everything after "/bench-abba" on its line, tokens in any order: a number = - # pair count; cont[TX]/mono[TX] = workload. + # Everything after "/bench-abba" on its line. The only token is an optional + # pair count; the workload is fixed to the real block. ARGS=$(printf '%s' "$COMMENT_BODY" | tr -d '\r' | sed -n 's|^/bench-abba||p' | head -n1) - PAIRS=20; CONTINUATIONS=1; TX_COUNT=100 + PAIRS=12 set -f # tokens must not glob-expand against the runner's CWD for tok in $ARGS; do case "$tok" in - cont) CONTINUATIONS=1; TX_COUNT=100 ;; - mono) CONTINUATIONS=0; TX_COUNT=5 ;; - cont[0-9]*) CONTINUATIONS=1; TX_COUNT="${tok#cont}" ;; - mono[0-9]*) CONTINUATIONS=0; TX_COUNT="${tok#mono}" ;; - [0-9]*) PAIRS="$tok" ;; - *) echo "::warning::ignoring unrecognized token '$tok'" ;; + [0-9]*) PAIRS="$tok" ;; + *) echo "::warning::ignoring unrecognized token '$tok'" ;; esac done - # Digits-only + clamps: PAIRS capped so one comment can't monopolize the - # single bench server; mono capped at 5tx (monolithic peak heap grows with - # the trace; 20tx needs ~78 GB). - if [ "$CONTINUATIONS" = "1" ]; then TX_DEFAULT=100; TX_MAX=100; else TX_DEFAULT=5; TX_MAX=5; fi - case "$TX_COUNT" in - ''|*[!0-9]*) echo "::warning::invalid tx count '$TX_COUNT'; using $TX_DEFAULT"; TX_COUNT=$TX_DEFAULT ;; - esac - if [ "$TX_COUNT" -lt 1 ] || [ "$TX_COUNT" -gt "$TX_MAX" ]; then - echo "::warning::tx count $TX_COUNT out of range [1,$TX_MAX] for this mode; using $TX_DEFAULT" - TX_COUNT=$TX_DEFAULT - fi + # Digits-only + clamp, so one comment cannot monopolize the single bench + # server: 40 pairs is already ~3.7 hr of it. case "$PAIRS" in - ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 20"; PAIRS=20 ;; + ''|*[!0-9]*) echo "::warning::invalid pair count '$PAIRS'; using 12"; PAIRS=12 ;; esac if [ "$PAIRS" -lt 2 ] || [ "$PAIRS" -gt 40 ]; then - echo "::warning::pair count $PAIRS out of range [2,40]; using 20" - PAIRS=20 + echo "::warning::pair count $PAIRS out of range [2,40]; using 12" + PAIRS=12 fi # Even is ideal so the AB/BA orders balance; round an odd request up by one. if [ "$((PAIRS % 2))" -ne 0 ]; then PAIRS=$((PAIRS + 1)) echo "::notice::rounded odd pair count up to $PAIRS so AB/BA orders balance" fi - if [ "$CONTINUATIONS" = "1" ]; then - WORKLOAD="ethrex ${TX_COUNT}tx continuations" - else - WORKLOAD="ethrex ${TX_COUNT}tx monolithic" - fi - # Outputs land before the fail-fast below so the always() result - # comment is fully labeled even when this step exits early. + # The block's identity lives in the Makefile; this job never names one. + WORKLOAD="ethrex real block, continuations" { echo "pairs=$PAIRS" - echo "continuations=$CONTINUATIONS" - echo "tx_count=$TX_COUNT" echo "workload=$WORKLOAD" } >> "$GITHUB_OUTPUT" - # Fail fast if the PR side predates the pointer_width_64 fix: a >=40tx - # continuation proof exceeds rkyv's old 2 GiB cap and only dies after - # blocking the single bench server for hours. Skip the check when the - # fetch fails: gh api prints HTTP error bodies to stdout, so gate on - # its exit status AND on the payload looking like the manifest (it - # declares rkyv) — never treat an error blob as a missing feature. - # Purely textual; deletable once every open branch postdates the fix. - if [ "$CONTINUATIONS" = "1" ] && [ "$TX_COUNT" -ge 40 ]; then - if SIDE=$(gh api "repos/$GITHUB_REPOSITORY/contents/prover/Cargo.toml?ref=$HEAD_SHA" \ - -H "Accept: application/vnd.github.raw" 2>/dev/null) \ - && printf '%s' "$SIDE" | grep -q '^rkyv' \ - && ! printf '%s' "$SIDE" | grep -q pointer_width_64; then - MSG="PR branch predates the rkyv pointer_width_64 fix — a ${TX_COUNT}tx continuation proof cannot serialize. Rebase onto main, or bench with cont10/mono." - echo "$MSG" > /tmp/abba_out.txt # surfaces in the result comment - echo "::error::$MSG" - exit 1 - fi - fi - echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD" + echo "Using $PAIRS A/B/B/A pairs on $WORKLOAD at epoch 2^$ABBA_REAL_EPOCH_LOG2" - name: Acknowledge (react + occupancy notice) uses: actions/github-script@v7 @@ -135,7 +117,9 @@ jobs: await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} (a cont100 pair is ~15 min, so the default 20 pairs runs ~4.5-5 hr; pass a smaller pair count or \`cont10\` for a quicker, coarser run). The bench server is occupied until it finishes.` + // A pair is TWO proves at the runner's measured 158.8 s, so ~5.3 min/pair, + // plus ~8 min of checkout, two-sided build and fixture fetch. + body: `⏳ **ABBA tiebreaker started** on the bench server: ${process.env.PAIRS} pairs of ${process.env.WORKLOAD} — a pair is 2 proves at ~158.8 s, so roughly ${Math.round(8 + Number(process.env.PAIRS) * 5.3)} min. Pass a smaller pair count for a quicker, coarser run. The bench server is occupied until it finishes.` }); - name: Checkout (full history for ref resolution) @@ -156,8 +140,12 @@ jobs: env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} - CONTINUATIONS: ${{ steps.cfg.outputs.continuations }} - TX_COUNT: ${{ steps.cfg.outputs.tx_count }} + # WORKLOAD=real is bench_abba.sh's own default; pinned here so the workload + # this job measures is stated in the job rather than inherited. The epoch is + # NOT the script's default (2^20, laptop-safe) — it is the runner's tier, the + # same one /bench proves at. + WORKLOAD: real + EPOCH_SIZE_LOG2: ${{ env.ABBA_REAL_EPOCH_LOG2 }} run: | export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" set -o pipefail diff --git a/.github/workflows/bench-verify.yml b/.github/workflows/bench-verify.yml index 23d616ec4..7774e0a6c 100644 --- a/.github/workflows/bench-verify.yml +++ b/.github/workflows/bench-verify.yml @@ -116,6 +116,12 @@ jobs: env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} PAIRS: ${{ steps.cfg.outputs.pairs }} + # The script defaults to the real block; this job pins the synthetic one. + # /bench-verify reports TWO arms and the real path runs only the continuation + # one (a real block does not fit monolithically), so inheriting the default + # would silently drop an arm and break comparability with every verify number + # recorded so far. + WORKLOAD: synthetic run: | export SYSROOT_DIR="$HOME/.lambda-vm-sysroot" set -o pipefail diff --git a/.github/workflows/benchmark-gpu.yml b/.github/workflows/benchmark-gpu.yml index 11fcbf222..b728e60f5 100644 --- a/.github/workflows/benchmark-gpu.yml +++ b/.github/workflows/benchmark-gpu.yml @@ -60,9 +60,11 @@ env: # # GPU PATH ONLY, and deliberately so. It is NOT pushed into bench_abba.sh's default # (2^20, which the CPU /bench-abba uses) nor into the CLI's - # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~28 GiB of HOST memory on - # a CPU build, which would break laptops. VRAM is the binding constraint here and host - # RAM is the binding constraint there, so the two defaults are not the same question. + # DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2 (also 20): 2^22 needs ~32 GiB of HOST memory on + # a CPU build (measured peak RSS on the 124 GiB calibration box; the CUDA path's host + # peak is a different number, ~36 GB — see the cpu_ram floor below), which would break + # laptops. VRAM is the binding constraint here and host RAM is the binding constraint + # there, so the two defaults are not the same question. GPU_REAL_EPOCH_LOG2: "22" # Unique per-run label set on the instance, for easy identification in the Vast console. RUN_LABEL: "gpu-bench-${{ github.run_id }}-${{ github.run_attempt }}" @@ -157,11 +159,12 @@ jobs: // Post the "started" notice under the SAME marker the result step uses, so the // result updates this comment in place (and re-runs reuse it rather than stacking). const marker = 'GPU Benchmark (ABBA)'; - // Reference: 4 pairs measured ~20 min end-to-end (rental + dual build + - // 8 proves). Per-prove wall varies with the rented host's CPU (the prover - // is partly host-CPU-bound), so scale expectations from pair count, not - // from a fixed per-run figure. - const mins = 12 + Number(process.env.PAIRS) * 2; + // Reference: 4 pairs measured 20 min 11 s end-to-end — 3 min 56 s of rental, + // checkout and dual cuda build, then 4.06 min per pair, since a pair is TWO + // proves at ~2 min each. Per-prove wall varies with the rented host's CPU + // (the prover is partly host-CPU-bound), so the slope is the measured one + // and the intercept carries slack for a colder box. + const mins = 12 + Number(process.env.PAIRS) * 4; const body = `## GPU Benchmark (ABBA) — running…\n\n⏳ Renting an RTX 5090 on Vast.ai and running ${process.env.PAIRS} interleaved pairs (PR vs main) of ${process.env.WORKLOAD} on the CUDA prover path. Rough ETA ~${mins} min. The result will replace this comment.`; const comments = await github.paginate(github.rest.issues.listComments, { owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index f9c83ea3b..cd72025a6 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -12,12 +12,13 @@ on: - 'executor/**' - 'bin/cli/**' - 'tooling/ethrex-fixtures/**' - # A baseline is only valid for the workload it measured, and these two - # define the real-block workload: the converter decides the fixture's - # bytes, the Makefile decides which block it is. Without them a repointed - # block would leave main's baseline stale until some prover file happened - # to change — and the comparison guard would suppress the table until then. - - 'tooling/ethrex-block-converter/**' + # A baseline is only valid for the workload it measured, and the Makefile is + # what defines that workload: it names the block and pins the URL and sha256 + # of the .bin this job fetches. Without it a repointed block would leave + # main's baseline stale until some prover file happened to change — and the + # comparison guard would suppress the table until then. The converter is not + # listed: CI never runs it, so changing it cannot change the bytes proven + # here. - 'Makefile' # Uncomment to auto-run on PRs: # pull_request: @@ -46,8 +47,8 @@ env: # 1. Its only unique coverage was the MONOLITHIC prove path, which is vestigial # (reportedly slower than a 1-epoch continuation). Coverage of a path we intend # to delete is not a reason to spend runner time. - # 2. Its crypto mix is one no real block has: 8.83 ECSM per Mcycle against a real - # block's ~1.3-1.6. Screening against it tunes the prover for a worst case that + # 2. Its crypto mix is one no real block has: 9.16 ECSM per Mcycle against a real + # block's 2.28. Screening against it tunes the prover for a worst case that # cannot occur. # # The synthetic fixtures themselves are NOT gone — /bench-growth still sweeps them to @@ -90,28 +91,31 @@ env: # 2^22 397.88 s 32.21 GiB RSS 13 epochs 1.15 GB proof # 2^23 356.47 s 60.01 GiB RSS 7 epochs 0.90 GB proof # - # This runner's floor is >=64 GiB, so 2^22's 32.2 GiB fits with ~50% headroom while - # 2^23's 60 GiB does not fit safely — memory, not speed, is what picks 2^22 here. - # Moving off 2^21 is worth ~14% wall on the calibration box. That box is NOT this - # runner (it measured 8.6 s/Mcycle against this runner's 5.3-5.6), so treat the - # ratio as transferable and the absolute seconds as not: the first /bench run at - # this epoch establishes this runner's actual time. + # Those RSS figures are the CALIBRATION BOX's, and memory does not transfer between + # machines any better than seconds do: this runner measured ~52 GB of peak heap for + # the same block at 2^22 — over 1.5x the calibration box's 32.21 GiB, and close + # enough to its >=64 GiB floor that 2^23 (60 GiB on the roomier box, so more here) + # is out of the question. Memory, not speed, is what picks 2^22. Moving off 2^21 is + # worth ~14% wall on the calibration box; treat that ratio as transferable and + # neither the absolute seconds nor the RSS column as such. This runner's own + # measured time at 2^22 is 158.8 s (median of 3, 2.8% spread, 13 epochs). # # Deliberately NOT the CLI's DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2, which stays 20 so # a laptop can still prove; and not the GPU path's 2^22, which happens to coincide # but is chosen by VRAM rather than host RAM (see benchmark-gpu.yml). REAL_BLOCK_EPOCH_LOG2: "22" - # Sampled like the synthetic screen, 3 runs, and reported the same way (median + - # spread). Fewer than the synthetic's 5 because each run is minutes rather than - # seconds; enough that one slow run shows up as spread instead of moving the median. + # Sampled 3 times and reported as median + spread. Three rather than five because + # each run is minutes rather than seconds; enough that one slow run shows up as + # spread instead of moving the median. # - # THIS IS THE DIAL. The runner's per-run real-block time is still unmeasured — the - # calibration ran on a different box (see REAL_BLOCK_EPOCH_LOG2 above). If the first - # runs land above ~6 min each, /bench becomes a ~25 min occupancy of a runner every - # other bench queues behind, and this count is what to turn down (2, or 1) before - # reaching for anything else. + # THIS IS THE DIAL. A run measures 158.8 s on this runner (median of 3, 2.8% spread, + # at the epoch above), so 3 runs is ~8 min of proving and the right count. Should a + # future block or prover change take a run past ~6 min, /bench becomes a ~25 min + # occupancy of a runner every other bench queues behind, and this count is what to + # turn down (2, or 1) before reaching for anything else. # `/bench N` overrides this, clamped to [1,5]. Past 5 the cached comparison can't - # beat the ~1% session-drift wall anyway — that is what /bench-abba is for. + # beat the ~1% session-drift wall anyway — that is what /bench-abba is for, and it + # proves this same block at this same epoch, so escalating keeps the question fixed. BENCH_RUNS_REAL: 3 # Memory-scaling sweep: same ELF, different N-transfer inputs. GROWTH_PROGRAMS # are the generated (gitignored) fixture basenames in executor/tests/; GROWTH_STEPS @@ -210,21 +214,24 @@ jobs: # The real block runs on EVERY invocation — plain `/bench`, push to main, and # workflow_dispatch alike. It is the number that means something, so it should - # not need a second command to ask for; the synthetic screen rides along - # unchanged as the cheap, high-sensitivity half. + # not need a second command to ask for, and it is the only workload /bench + # proves. # - # The cost is real and lands on a shared runner: ~4.5-4.8 min per run x - # BENCH_RUNS_REAL, so every /bench and every push to main now occupies the - # bench server for roughly 15-20 min, and /bench-abba and /bench-verify queue - # behind it. That trade was made deliberately — see BENCH_RUNS_REAL above for - # the dial if it proves too expensive. + # The cost is real and lands on a shared runner: 158.8 s per run x + # BENCH_RUNS_REAL is ~8 min of proving, so every /bench and every push to main + # occupies the bench server for roughly 15 min once checkout, the two-sided + # build, the fixture fetch and the guest ELF are counted, and /bench-abba and + # /bench-verify queue behind it. That trade was made deliberately — see + # BENCH_RUNS_REAL above for the dial if it proves too expensive. RUN_REAL=true # The fixture is FETCHED from a pinned URL + sha256, not built (see the # Makefile). The guard covers the window after a repoint but before the new - # artifact is uploaded: degrade to the synthetic sections with a warning - # rather than failing, so a push to main cannot go red over an upload nobody - # in CI can perform. + # artifact is uploaded: warn instead of failing, so a push to main cannot go + # red over an upload nobody in CI can perform. On push and workflow_dispatch + # the growth sweep still runs and feeds the baseline artifact (no comment is + # posted on those events — the Comment step is gated to comment triggers); + # on a plain /bench nothing is left to measure, and the footer says so. if [ "$RUN_REAL" = "true" ] && [ -z "$(make -s print-real-block-fixture-url)" ]; then echo "::warning::Real-block benchmark skipped: ETHREX_REAL_BLOCK_FIXTURE_URL is unset in the Makefile." RUN_REAL=false @@ -237,9 +244,9 @@ jobs: # workload even when main's Makefile names a different block. echo "REAL_INPUT=$(make -s print-real-block-fixture)" >> "$GITHUB_ENV" - # Every metrics writer appends; the step that used to create this file was the - # synthetic screen, which is gone. Truncate here (this step always runs) so a - # previous run's values can't survive on the persistent self-hosted runner. + # Every metrics writer appends, and no step creates this file. Truncate here + # (this step always runs) so a previous run's values can't survive on the + # persistent self-hosted runner. : > /tmp/metrics.txt # `/bench N` sets the real-block sample count. Both sides use the same count @@ -253,8 +260,9 @@ jobs: fi # Clamp to 1-5. Beyond 5 the single-session cached comparison barely improves - # (it can't beat the ~1% drift wall); use the ABBA tiebreaker for finer deltas. - # At ~4-5 min a run this is also the difference between a 15 min and a 25 min + # (it can't beat the ~1% drift wall); use /bench-abba for finer deltas — it + # resolves ~1% over the same block in ~72 min at its default 12 pairs. + # At 158.8 s a run this is also the difference between a ~10 min and a ~20 min # occupancy of the one bench runner. if [ "$RUNS" -lt 1 ] 2>/dev/null || [ "$RUNS" -gt 5 ] 2>/dev/null; then echo "::warning::Run count $RUNS out of range [1,5], defaulting to $BENCH_RUNS_REAL" @@ -410,9 +418,9 @@ jobs: EPOCHS="$E" done - # Same summary as the synthetic screen: median for the headline, spread so a - # single slow run is visible rather than silently shifting the verdict, and - # every raw value so a reader can judge for themselves. + # Median for the headline, spread so a single slow run is visible rather than + # silently shifting the verdict, and every raw value so a reader can judge for + # themselves. MEDIAN_POS=$(( (RUNS + 1) / 2 )) TIME_MEDIAN=$(echo $TIMES | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") HEAP_MEDIAN=$(echo $HEAPS | tr ' ' '\n' | sort -n | awk "NR==$MEDIAN_POS") @@ -563,7 +571,7 @@ jobs: echo "growth_r2=$R2" >> "$GITHUB_OUTPUT" fi # end run_growth check - # --- Real block (continuations, 1 sample) --- + # --- Real block (continuations, sampled to the PR side's count) --- # Only reached when no cached baseline exists, which is the expensive case: # push-to-main normally publishes the real-block numbers, so a /bench comment # pays for the PR side alone. $REAL_INPUT was resolved before this @@ -703,6 +711,19 @@ jobs: fi fi + # Growth comparison. The renderer keys the whole growth section off + # pr_growth_heaps, so these pass through even when only one side ran. + { + echo "pr_growth_heaps=$PR_GROWTH_HEAPS" + echo "pr_growth_times=$PR_GROWTH_TIMES" + echo "pr_growth_slope=$PR_GROWTH_SLOPE" + echo "pr_growth_r2=$PR_GROWTH_R2" + echo "baseline_growth_heaps=$BASELINE_GROWTH_HEAPS" + echo "baseline_growth_times=$BASELINE_GROWTH_TIMES" + echo "baseline_growth_slope=$BASELINE_GROWTH_SLOPE" + echo "baseline_growth_r2=$BASELINE_GROWTH_R2" + } >> "$GITHUB_OUTPUT" + # Growth slope comparison if [ -n "$BASELINE_GROWTH_SLOPE" ] && [ -n "$PR_GROWTH_SLOPE" ]; then SLOPE_DIFF=$((PR_GROWTH_SLOPE - BASELINE_GROWTH_SLOPE)) @@ -746,7 +767,7 @@ jobs: with: # Renderable offline: `node scripts/render_bench_comment.js` extracts this block # and prints the markdown for a set of scenarios, so wording and formatting can - # be checked without occupying the bench server for 15-20 min. Add a scenario + # be checked without occupying the bench server for ~15 min. Add a scenario # there when you add a branch here. script: | const baseSrc = process.env.BASELINE_SRC; @@ -791,8 +812,8 @@ jobs: // --- Section 1: Real block (headline; present whenever the fixture is fetchable) --- if (realTime) { body += `## Benchmark — real block${realInput ? ` (\`${realInput}\`)` : ''}${parseInt(realRuns) > 1 ? ` (median of ${realRuns})` : ''}\n\n`; - // Run count lives in the heading ("median of N"), as on the synthetic - // section — repeating it here just made the two disagree in tone. + // Run count lives in the heading ("median of N"); the subheading carries + // only the prove configuration. body += `continuations · epoch 2^${realEpochLog2}`; if (realEpochs) body += ` · ${realEpochs} epochs`; body += `\n\n`; @@ -810,16 +831,18 @@ jobs: } body += `| **Prove time** | ${baseRealTime}s | ${realTime}s | ${fmt(realTimeDiff)}s (${fmt(realTimePct)}%) ${icon(realTimePct)} |\n\n`; - // Wider bands than the synthetic screen's 5%: 3 runs of a minutes-long - // prove resolve less than 5 runs of a 25s one, so the middle is reported - // as unresolved rather than as "fine". + // Bands of 10%/3%, wider than a fast workload would need: 3 runs of a + // minutes-long prove resolve coarsely, so the middle is reported as + // unresolved rather than as "fine". const rp = parseFloat(realTimePct); if (rp > 10) { body += `> ⚠️ **Regression on the real block** — prove time up ${Math.abs(rp).toFixed(1)}%.\n`; } else if (rp < -10) { body += `> 🎉 **Improvement on the real block** — prove time down ${Math.abs(rp).toFixed(1)}%.\n`; } else if (Math.abs(rp) >= 3) { - body += `> ❓ **${fmt(realTimePct)}% — beyond what ${realRuns} runs resolve.** Use \`/bench-abba\` for a paired test.\n`; + // /bench-abba proves this same block at this same epoch, so the + // escalation resolves the reading rather than changing the question. + body += `> ❓ **${fmt(realTimePct)}% — beyond what ${realRuns} runs resolve.** Use \`/bench-abba\` for a paired test of the same block (default 12 pairs, ~72 min, resolves ~1%).\n`; } else { body += `> ✅ No significant change.\n`; } @@ -841,7 +864,7 @@ jobs: body += `\n`; } - // --- Section 3: Memory growth --- + // --- Section 2: Memory growth (only when the growth sweep ran) --- // Synthetic on purpose: this plots heap against BLOCK SIZE, which needs a // family of blocks that differ only in transaction count. A real block is // one point and cannot produce a slope. diff --git a/.github/workflows/pr_main.yaml b/.github/workflows/pr_main.yaml index 5d560a63a..08879ec3f 100644 --- a/.github/workflows/pr_main.yaml +++ b/.github/workflows/pr_main.yaml @@ -40,6 +40,14 @@ jobs: - name: Check ethrex fixture checksums run: make check-ethrex-fixture-checksums + # The benchmark comment harness: verifies every steps.X.outputs.Y consumed in + # benchmark-pr.yml has a producer (the orphaned-wiring class that once broke + # /bench-growth silently), then renders the full comment scenario suite. + # Pure node, no dependencies, ~10 s. Lives in Lint so it runs on every PR — + # a path filter would have to enumerate every file that can break the wiring. + - name: Check benchmark comment wiring + run: node scripts/render_bench_comment.js + test-executor: name: Executor tests runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index a3471bdb4..25dce43de 100644 --- a/Makefile +++ b/Makefile @@ -282,7 +282,8 @@ test-rust: compile-programs-rust # # the fixture the rkyv ProgramInput the benchmarks prove (~1 MB) # the cache the ethrex-replay JSON it was converted from (~2 MB), read only -# by the converter's tests and by `regen-real-block-fixture` +# by `regen-real-block-fixture`. The converter's TESTS read a +# different, upstream-pinned cache — see below. # # Fetching a verified binary is the same contract as prepare-sysroot above, and it # keeps the converter, the ~335-package ethrex host dependency tree and an @@ -296,7 +297,7 @@ test-rust: compile-programs-rust # below. It is a regeneration tool for ethrex rev bumps, not a build step. # # ---- Repointing to a different block ---- -# These FIVE lines and nothing else. Every path below derives from them, the +# These SIX lines and nothing else. Every path below derives from them, the # benchmark scripts and CI resolve the fixture through # `make -s print-real-block-fixture`, and no workflow, script or env var anywhere # names a block. Outside this file the repoint touches only REAL_BLOCK_FIXTURE in @@ -377,9 +378,10 @@ ethrex-real-block-cache: $(call ensure_verified,$(ETHREX_REAL_BLOCK_CACHE_URL),$(ETHREX_REAL_BLOCK_CACHE_SHA256),$(ETHREX_REAL_BLOCK_CACHE),cache,ETHREX_REAL_BLOCK_CACHE_URL) # Single source of truth for the benchmark tooling. scripts/bench_verify.sh, -# scripts/perf_diff.sh and .github/workflows/benchmark-pr.yml read the fixture -# path from here instead of hardcoding it, so repointing the block above moves -# every benchmark at once. `-s` on the caller's side keeps the output clean. +# scripts/bench_abba.sh, scripts/perf_diff.sh and +# .github/workflows/benchmark-pr.yml read the fixture path from here instead of +# hardcoding it, so repointing the block above moves every benchmark at once. +# `-s` on the caller's side keeps the output clean. print-real-block-fixture: @echo $(ETHREX_REAL_BLOCK_FIXTURE) @@ -443,8 +445,11 @@ regen-real-block-fixture: ethrex-real-block-cache # ethrex host-reference tests live in the detached `tooling/ethrex-tests` # workspace (ethrex pins rkyv's `unaligned` feature; isolated Cargo.lock). -# Needs the real-block fixture, so it needs the fixture URL to be set; CI runs the -# `-offline` variant below in the PR gate and this one only in ethrex-block-converter.yml. +# Needs the real-block fixture, so it needs the fixture URL to be set. This is a +# local convenience target: no workflow invokes it. The PR gate spells out the +# `-offline` variant below inline (pr_main.yaml), and ethrex-block-converter.yml's +# block-usability job fetches the fixture and runs +# `test_ethrex_real_block_native` on its own. test-ethrex: compile-programs-rust ethrex-real-block-fixture cd tooling/ethrex-tests && cargo test --release -- --include-ignored --skip test_ethrex_real_block_vm diff --git a/executor/.gitignore b/executor/.gitignore index 1229bde5c..28b14d098 100644 --- a/executor/.gitignore +++ b/executor/.gitignore @@ -1,7 +1,8 @@ /target /program_artifacts/rust -# Real-block fixtures (~1 MB): generated by `make ethrex-real-block-fixture` -# from an ethrex-replay cache, never committed. See tooling/ethrex-block-converter. +# Real-block fixtures (~1 MB): fetched by `make ethrex-real-block-fixture` from a +# release asset and verified against a sha256 pinned in the Makefile, never +# committed. See tooling/ethrex-block-converter. # One pattern per network the converter accepts (mainnet/hoodi/sepolia — it # rejects anything else), so repointing ETHREX_REAL_BLOCK_NETWORK in the Makefile # cannot silently make a ~1 MB fixture committable. diff --git a/executor/tests/README.md b/executor/tests/README.md index 80c16d4eb..fdbf47bcb 100644 --- a/executor/tests/README.md +++ b/executor/tests/README.md @@ -57,8 +57,11 @@ ethrex_10_transfers.bin The blocks above are synthetic (N plain ETH transfers over a small genesis). For a representative workload — real contract execution, real trie depth, real -bytecode — `make ethrex-real-block-fixture` generates -`ethrex_hoodi_1265656.bin` from an ethrex-replay cache. It is ~1 MB, so it is -gitignored and generated on demand rather than committed, and its checksum is -pinned in `tooling/ethrex-block-converter/README.md` rather than above (the -checksum script only covers committed fixtures). +bytecode — `make ethrex-real-block-fixture` downloads +`ethrex_mainnet_25368371.bin` (1,110,156 B) from the `bench-fixtures-v1` release +and verifies it against `ETHREX_REAL_BLOCK_FIXTURE_SHA256` in the Makefile before +moving it into place. It is gitignored rather than committed, so the checksum +lives next to the URL in the Makefile rather than in the table above (the checksum +script only covers committed fixtures). See +`tooling/ethrex-block-converter/README.md` for how the fixture is produced and +repointed. diff --git a/scripts/bench_abba.sh b/scripts/bench_abba.sh index 061c6e407..0baa21db7 100755 --- a/scripts/bench_abba.sh +++ b/scripts/bench_abba.sh @@ -37,22 +37,33 @@ # TX_COUNT= ethrex transfer fixture to prove (default 5; use 20 for a # large continuation trace where GPU-residency wins are visible). # Ignored when WORKLOAD=real. -# WORKLOAD=synthetic|real (default synthetic) which block to prove. `real` +# WORKLOAD=real|synthetic (default real) which block to prove. `real` # fetches the real-block fixture (block identity lives in the Makefile) and # forces --continuations; TX_COUNT and CONTINUATIONS do not apply to it. # -# On WORKLOAD: the synthetic default is N plain transfers — ecrecover-heavy over a -# near-empty state — while a real block is keccak- and trie-bound, so a prover change -# can move the two in opposite directions. `real` is what benchmark-gpu.yml runs by -# default; `synthetic` stays the default HERE so CPU /bench-abba results remain -# comparable with everything recorded before. +# On WORKLOAD: a real block is keccak- and trie-bound, while the synthetic option is +# N plain transfers — ecrecover-heavy over a near-empty state — so a prover change can +# move the two in opposite directions. `real` is the default here, in benchmark-gpu.yml +# and in /bench, so the screen and its tiebreaker resolve the same workload; pass +# WORKLOAD=synthetic to reproduce a number recorded against that fixture. # -# Sizing (ethrex pair-noise sd ~1.2%, 80% power): ~12 pairs for a 1% effect, -# ~18 for 0.8%, ~32 for 0.6%. Default 20 -> solid on 0.8-1%, ~60% power at 0.6% -# (if a 20-pair run straddles 0 on a ~0.6%-looking effect, extend to 32). +# Sizing at WORKLOAD=real, from the paired t-test (resolvable 95% delta = +# t* x sd / sqrt(N)). The pair-delta sd on the bench runner is NOT yet measured; the +# two columns bracket it between 1.0% (the GPU box's measured 0.64% plus margin) and +# 2.0% (sqrt(2) x the runner's measured 1.43% single-run CV): +# +# pairs wall resolves (sd 2.0% / sd 1.0%) +# 8 ~50 min 1.7% / 0.8% +# 12 ~72 min 1.3% / 0.6% <- workflow default +# 20 ~1h55m 0.9% / 0.5% +# 32 ~3h 0.7% / 0.4% +# +# Wall assumes epoch 2^22 (158.8 s per prove, two per pair) plus ~8 min of setup. +# The first real ABBA run MEASURES that sd — read it off the `sd` field of the +# paired-t line printed below — and this table should be re-pinned to it. # # scripts/bench_abba.sh origin/my-pr-branch # vs main, 20 pairs -# scripts/bench_abba.sh origin/my-pr-branch origin/main 32 # 32 pairs (~0.6%) +# scripts/bench_abba.sh origin/my-pr-branch origin/main 32 # 32 pairs set -euo pipefail @@ -76,7 +87,7 @@ CONTINUATIONS="${CONTINUATIONS:-0}" EPOCH_SIZE_LOG2="${EPOCH_SIZE_LOG2:-20}" # ethrex transfer-count fixture to prove (executor/tests/ethrex_${TX_COUNT}_transfers.bin). TX_COUNT="${TX_COUNT:-5}" -WORKLOAD="${WORKLOAD:-synthetic}" +WORKLOAD="${WORKLOAD:-real}" case "$WORKLOAD" in synthetic|real) ;; *) echo "ERROR: WORKLOAD must be 'synthetic' or 'real' (got '$WORKLOAD')." >&2; exit 2 ;; @@ -136,18 +147,18 @@ if [ ! -f "$ELF_REL" ]; then export SYSROOT_DIR="${SYSROOT_DIR:-$HOME/.lambda-vm-sysroot}" make "$ELF_REL" fi -if [ ! -f "$INPUT_REL" ]; then - if [ "$WORKLOAD" = "real" ]; then - # ~1 MB, gitignored, never in a fresh checkout — and a rented GPU box is always a - # fresh checkout. Fetched by URL + sha256, not built: no converter, no ethrex host - # dependency tree, so this costs seconds on the box. - echo "==> Fetching ethrex real-block fixture (missing)" - make ethrex-real-block-fixture - else - echo "==> Generating ethrex ${TX_COUNT}-transfer fixture (missing)" - ( cd tooling/ethrex-fixtures && cargo build --release ) - tooling/ethrex-fixtures/target/release/ethrex-fixtures "$TX_COUNT" "$INPUT_REL" distinct - fi +if [ "$WORKLOAD" = "real" ]; then + # ~1 MB, gitignored, never in a fresh checkout — and a rented GPU box is always a + # fresh checkout. Fetched by URL + sha256, not built: no converter, no ethrex host + # dependency tree, so this costs seconds on the box. Unconditional on purpose: the + # target hashes whatever is on disk on every invocation, which is what catches a + # copy left behind by an earlier run in the same rental. A match costs ~35 ms. + echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)" + make ethrex-real-block-fixture +elif [ ! -f "$INPUT_REL" ]; then + echo "==> Generating ethrex ${TX_COUNT}-transfer fixture (missing)" + ( cd tooling/ethrex-fixtures && cargo build --release ) + tooling/ethrex-fixtures/target/release/ethrex-fixtures "$TX_COUNT" "$INPUT_REL" distinct fi ELF="$(cd "$(dirname "$ELF_REL")" && pwd)/$(basename "$ELF_REL")" INPUT="$(cd "$(dirname "$INPUT_REL")" && pwd)/$(basename "$INPUT_REL")" diff --git a/scripts/bench_verify.sh b/scripts/bench_verify.sh index 068b4c864..369041c48 100755 --- a/scripts/bench_verify.sh +++ b/scripts/bench_verify.sh @@ -4,12 +4,15 @@ # Reported % = (PR - baseline)/baseline, matching the classic /bench: # NEGATIVE numbers are improvements (PR faster/smaller); positive = regression. # -# TWO arms over the same ethrex 20-tx block, both at blowup=2 / 219 queries: +# Arms over one block, both at blowup=2 / 219 queries: # monolithic one VmProof for the whole execution. # continuations the same block proved as 2^CONT_EPOCH_LOG2-cycle epochs and verified # as a ContinuationProof bundle — what /bench proves and what an L2 # actually runs, so a verifier change that only moves per-epoch or # aggregation cost is invisible in the monolithic arm alone. +# WORKLOAD picks the block: `synthetic` (default) is the ethrex 20-tx one and runs both +# arms; `real` runs the continuation arm only, since a real block does not fit +# monolithically. See "Workload" below. # The continuation arm is best-effort: if its prove or verify fails (it is the # memory-hungry one) the arm is skipped with a note and the monolithic verdict still # posts, rather than failing the whole bench. @@ -27,34 +30,32 @@ # below 6: the exact Wilcoxon's smallest attainable two-sided p is 2/2^n, so at # n=4 it is 0.125 and the arm can only ever report BORDERLINE, however large and # clean the effect. -# WORKLOAD=synthetic|real (default synthetic) which BLOCK both arms prove. +# WORKLOAD=real|synthetic (default real) which BLOCK both arms prove. # `real` fetches the real-block fixture (identity lives in the Makefile) and runs # the continuation arm ONLY — a real block is hundreds of GB monolithically, so # that arm is skipped rather than left to OOM. See "Workload" below. -# CONT_EPOCH_LOG2= continuation epoch size (default 20, min 18). For -# WORKLOAD=real prefer the calibrated tier for the box you are on — 2^22 on the -# bench runner or a 64 GiB machine, 2^23 on a 128 GiB one (see -# tooling/ethrex-block-converter/README.md, "Choosing the epoch size"); the default -# below is chosen for the SYNTHETIC arm. 20 matches -# scripts/bench_abba.sh, so this arm proves the same bundle shape /bench already -# proves on the same server — and 20 txs at 2^20 is strictly cheaper than the -# 100 txs at 2^20 that /bench runs by default, so it can't be the thing that OOMs -# the box. (`cli prove --epoch-size-log2 --help` measured ethrex 10tx at ~9.5 GB -# for 2^20 vs ~15.8 GB for 2^21.) Note this does NOT match -# bench_recursion_cycles.sh's BLOCK_EPOCH_LOG2=21: that arm needs FEW epochs so -# the bundle fits the guest's 512 MiB private-input cap, a constraint that -# doesn't apply to host-side verification. +# CONT_EPOCH_LOG2= continuation epoch size (default 20, min 18). 20 is the +# laptop-safe setting, not the fast one: prefer the calibrated tier for the box +# you are on — 2^22 on the bench runner or a 64 GiB machine, 2^23 on a 128 GiB +# one (see tooling/ethrex-block-converter/README.md, "Choosing the epoch size"), +# which is what /bench and /bench-abba pin. (`cli prove --epoch-size-log2 --help` +# measured ethrex 10tx at ~9.5 GB for 2^20 vs ~15.8 GB for 2^21.) Note this does +# NOT match bench_recursion_cycles.sh's BLOCK_EPOCH_LOG2=21: that arm needs FEW +# epochs so the bundle fits the guest's 512 MiB private-input cap, a constraint +# that doesn't apply to host-side verification. # # Workload. What this script measures is VERIFY cost, and verify cost is structural in # the proof — table mix, trace lengths, query count — so the block decides what is being -# measured, not the loop around it. The synthetic default is 20 plain transfers: -# ecrecover-heavy over a near-empty state. A real block inverts that mix (keccak- and -# trie-bound), so a verifier change can move the two differently. +# measured, not the loop around it. A real block is keccak- and trie-bound; the synthetic +# option is 20 plain transfers, ecrecover-heavy over a near-empty state, and inverts that +# mix, so a verifier change can move the two differently. # -# `synthetic` stays the default because it is what `/bench-verify` runs and what every -# number recorded so far used; `real` is the representative one, and costs a ~4.5-4.8 -# min continuation prove per side (cached in $WORK afterwards) before any verify run. -# Both sides always prove the same block, so a comparison is never mixed. +# `real` is the default here because it is the representative one and what /bench and +# /bench-abba measure. It costs a ~2.6 min continuation prove per side (cached in $WORK +# afterwards) before any verify run, and skips the monolithic arm. `/bench-verify` pins +# WORKLOAD=synthetic so its two-arm reports stay comparable with the numbers recorded +# against that fixture. Both sides always prove the same block, so a comparison is +# never mixed. set -euo pipefail @@ -69,7 +70,7 @@ N_PAIRS="${3:-20}" BENCH_FEATURES="${BENCH_FEATURES:-jemalloc-stats}" CONT_PAIRS="${CONT_PAIRS:-8}" CONT_EPOCH_LOG2="${CONT_EPOCH_LOG2:-20}" -WORKLOAD="${WORKLOAD:-synthetic}" +WORKLOAD="${WORKLOAD:-real}" case "$WORKLOAD" in synthetic|real) ;; *) echo "ERROR: WORKLOAD must be 'synthetic' or 'real' (got '$WORKLOAD')." >&2; exit 2 ;; @@ -179,17 +180,18 @@ if [ ! -f "$ELF_REL" ]; then export SYSROOT_DIR="${SYSROOT_DIR:-$HOME/.lambda-vm-sysroot}" make "$ELF_REL" fi -if [ ! -f "$INPUT_REL" ]; then - if [ "$WORKLOAD" = "real" ]; then - # ~1 MB, gitignored, never in a fresh checkout. Fetched by URL + sha256, not built: - # no converter and no ethrex host dependency tree on this path. - echo "==> Fetching ethrex real-block fixture (missing)" - make ethrex-real-block-fixture - else - echo "==> Generating ethrex 20-transfer fixture (missing)" - ( cd tooling/ethrex-fixtures && cargo build --release ) - tooling/ethrex-fixtures/target/release/ethrex-fixtures 20 "$INPUT_REL" distinct - fi +if [ "$WORKLOAD" = "real" ]; then + # ~1 MB, gitignored, never in a fresh checkout. Fetched by URL + sha256, not built: + # no converter and no ethrex host dependency tree on this path. Unconditional on + # purpose: the target hashes whatever is already on disk on every invocation, which + # is how a stale copy left by an earlier block or an interrupted fetch gets caught. + # A match costs ~35 ms, so there is nothing to gate it on. + echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)" + make ethrex-real-block-fixture +elif [ ! -f "$INPUT_REL" ]; then + echo "==> Generating ethrex 20-transfer fixture (missing)" + ( cd tooling/ethrex-fixtures && cargo build --release ) + tooling/ethrex-fixtures/target/release/ethrex-fixtures 20 "$INPUT_REL" distinct fi ELF="$(cd "$(dirname "$ELF_REL")" && pwd)/$(basename "$ELF_REL")" INPUT="$(cd "$(dirname "$INPUT_REL")" && pwd)/$(basename "$INPUT_REL")" diff --git a/scripts/perf_diff.sh b/scripts/perf_diff.sh index a6c0380be..2c359a548 100755 --- a/scripts/perf_diff.sh +++ b/scripts/perf_diff.sh @@ -13,22 +13,23 @@ # # USAGE (on the bench server): # scripts/perf_diff.sh REF_A [REF_B=origin/main] -# Env: WORKLOAD=synthetic|real (default synthetic) picks the block to profile; +# Env: WORKLOAD=real|synthetic (default real) picks the block to profile; # EPOCH_SIZE_LOG2= (default 22) sizes the epoch, WORKLOAD=real only. # 22 is the calibrated bench-runner tier, matching /bench; use 23 on a # 128 GiB box (tooling/ethrex-block-converter/README.md, "Choosing the epoch size"). # # Pick the workload that matches the run you are localizing, because the symbol -# mix follows the block: the synthetic default is 20 plain transfers (8.73M cycles, -# 411 keccak calls, 80 ecsm calls) and a real block inverts that (50.78M cycles, -# 10,478 keccak, 116 ecsm), so a hot symbol in one need not be hot in the other. +# mix follows the block: the real default is 50.78M cycles, 10,478 keccak calls and +# 116 ecsm calls, and the synthetic option (20 plain transfers) inverts that at +# 8.73M cycles, 411 keccak, 80 ecsm — so a hot symbol in one need not be hot in the other. # Both counts are from the same guest ELF (merge fdb92f67, main @ 9ccdaf2, clang 21); # they move with guest optimisation (#861's thin LTO) and ~2% with the clang major, so # pin the ELF when quoting one. # # WORKLOAD=real also switches to a continuation prove (monolithic would need ~240 GB -# at that trace length), which is ~4-5 min per recording — five recordings, so budget -# ~25 min, plus ~1.2 GB of disk per bundle and ~32 GiB of RAM at the default epoch. +# at that trace length), which is 158.8 s per recording on the bench runner — five +# recordings, so budget ~13 min of proving, plus ~1.2 GB of disk per bundle and ~52 GB +# of RAM at the default epoch. # # Produces: # - two perf-diff tables (recorded twice per side, interleaved B A B A — @@ -46,10 +47,10 @@ if [ $# -lt 1 ]; then fi REF_A="$1" REF_B="${2:-origin/main}" -WORKLOAD="${WORKLOAD:-synthetic}" +WORKLOAD="${WORKLOAD:-real}" # 2^22: the calibrated tier for the bench server this script targets, same as -# /bench's real-block arm. Memory picks it, not speed — 32.2 GiB fits a >=64 GiB box with ~50% -# headroom where 2^23's 60 GiB does not. +# /bench's real-block arm. Memory picks it, not speed — that server peaks at ~52 GB on +# a >=64 GiB floor, and 2^23 measured 60 GiB on a roomier box, so it would not fit here. EPOCH_SIZE_LOG2="${EPOCH_SIZE_LOG2:-22}" case "$WORKLOAD" in synthetic|real) ;; @@ -76,16 +77,16 @@ fi command -v perf >/dev/null 2>&1 || { echo "ERROR: perf not installed (linux-tools)." >&2; exit 1; } [ -f "$ELF_REL" ] || { echo "ERROR: missing $ELF_REL — run bench_abba.sh once (it builds the guest)." >&2; exit 1; } -if [ ! -f "$INPUT_REL" ]; then - if [ "$WORKLOAD" = "real" ]; then - # ~1 MB, gitignored, never in a fresh checkout; fetch rather than abort. This is - # a URL + sha256 download, not a build. - echo "==> Fetching ethrex real-block fixture (missing)" - make ethrex-real-block-fixture - else - echo "ERROR: missing $INPUT_REL — run bench_abba.sh once (it builds the fixture)." >&2 - exit 1 - fi +if [ "$WORKLOAD" = "real" ]; then + # ~1 MB, gitignored, never in a fresh checkout; fetch rather than abort. This is a + # URL + sha256 download, not a build. Unconditional on purpose: the target hashes + # whatever is on disk on every invocation, which is how a stale copy gets caught. + # A match costs ~35 ms. + echo "==> Verifying ethrex real-block fixture (fetches on a digest miss)" + make ethrex-real-block-fixture +elif [ ! -f "$INPUT_REL" ]; then + echo "ERROR: missing $INPUT_REL — run WORKLOAD=synthetic scripts/bench_abba.sh once (it generates the synthetic fixture; the default workload is real and would not)." >&2 + exit 1 fi echo "==> Workload: $WORKLOAD ($INPUT_REL${CONT_ARGS:+, continuations epoch 2^$EPOCH_SIZE_LOG2})" diff --git a/scripts/render_bench_comment.js b/scripts/render_bench_comment.js index eee39f231..abc994f4c 100644 --- a/scripts/render_bench_comment.js +++ b/scripts/render_bench_comment.js @@ -4,7 +4,7 @@ // // WHY: that comment is ~200 lines of JS embedded in YAML, and the only other way to // see its output is to push and run a real benchmark on the shared bench server -// (~15-20 min of occupancy). This extracts the `script:` block from the workflow, +// (~15 min of occupancy). This extracts the `script:` block from the workflow, // runs it against synthetic env values, and prints the markdown — so a wording or // formatting change is checked in a second. // @@ -12,6 +12,11 @@ // that have actually broken before: sign handling on the verdict lines, the // no-baseline and block-mismatch paths, and the fixture-unavailable footer. // +// The one thing it does check is step wiring, because the scenarios cannot: they +// hand env to the script directly, so a `steps.X.outputs.Y` that no step writes +// still renders perfectly here while arriving empty in CI — which drops a whole +// section with no error anywhere. That check runs first and exits non-zero. +// // USAGE: node scripts/render_bench_comment.js // // Add a scenario by adding a key to `scenarios`; each is the env the comment step @@ -33,6 +38,47 @@ for (let i = start + 1; i < lines.length; i++) { } const src = body.join('\n'); +// --- Wiring check: every consumed step output has a producer -------------------- +// Steps are ` - name:` at six spaces; a step's outputs are the `key=` it echoes +// into $GITHUB_OUTPUT. A step that builds key names at runtime (`echo "$key=..."`) +// cannot be enumerated statically, so it is skipped rather than guessed at. +const stepStart = (l) => /^ {6}- name:/.test(l); +const produced = new Map(); // id -> Set of statically enumerable output keys +const allIds = new Set(); // every step id, so a typoed reference cannot pass +const opaque = new Set(); // outputs not statically enumerable: `uses:` actions + // (native outputs) and runtime-built `echo "$key=..."` +for (let i = 0; i < lines.length; i++) { + if (!stepStart(lines[i])) continue; + let end = i + 1; + while (end < lines.length && !stepStart(lines[end])) end++; + const block = lines.slice(i, end).join('\n'); + const id = (block.match(/^ {8}id: (\S+)$/m) || [])[1]; + if (!id) continue; + allIds.add(id); + if (/^ {8}uses:/m.test(block) || /echo "\$\{?[A-Za-z_][^"]*=/.test(block)) { + opaque.add(id); + continue; + } + // A run: step with no $GITHUB_OUTPUT produces nothing — reads from it must fail. + produced.set(id, new Set( + block.includes('$GITHUB_OUTPUT') + ? [...block.matchAll(/echo "([a-z0-9_]+)=/g)].map(m => m[1]) + : [] + )); +} +const problems = []; +for (const m of wf.matchAll(/steps\.([a-z0-9_-]+)\.outputs\.([a-z0-9_]+)/g)) { + const [ref, id, key] = m; + if (!allIds.has(id)) problems.push(`${ref} (no step with id '${id}' — typo?)`); + else if (opaque.has(id)) continue; // uses:/dynamic — cannot verify statically + else if (!produced.get(id).has(key)) problems.push(`${ref} (step '${id}' never writes '${key}')`); +} +if (problems.length) { + console.error(`Unwired step outputs:\n ${[...new Set(problems)].join('\n ')}`); + process.exit(1); +} +console.log(`Wiring OK: ${produced.size} producer steps verified, ${opaque.size} opaque (uses:/dynamic) skipped, no unknown step ids.`); + const captured = []; const github = { rest: { issues: { listComments: async () => ({ data: [] }), @@ -57,8 +103,11 @@ const scenarios = { REAL_TIME_DIFF: '-58.600', REAL_TIME_PCT: '-17.2', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0', }, 'C: clear regression': { + // Diffs are unsigned as the Compare step prints them (awk "%.3f"/"%.1f", `$((a-b))`); + // the renderer's own `fmt` adds the sign, so feeding one signed here would test + // a format CI never produces. ...REAL, BASE_REAL_TIME: '240.000', BASE_REAL_PEAK: '32975', - REAL_TIME_DIFF: '+41.400', REAL_TIME_PCT: '17.3', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0', + REAL_TIME_DIFF: '41.400', REAL_TIME_PCT: '17.3', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0', }, 'D: unresolved middle band (3-10%)': { ...REAL, BASE_REAL_TIME: '300.000', BASE_REAL_PEAK: '32975', @@ -75,7 +124,9 @@ const scenarios = { REAL_MISMATCH: 'ethrex_hoodi_1265656.bin', }, 'H: fixture unavailable': { COMMIT_SHA: 'c4e42b2900', REAL_EPOCH_LOG2: '22', BASELINE_SRC: 'cached' }, - 'I: with growth sweep (push-style)': { + // The growth sweep reaches a comment only on /bench-growth: run_growth is also true + // on push and workflow_dispatch, but the Comment step is skipped for both. + 'I: /bench-growth — real block plus the growth sweep': { ...REAL, BASE_REAL_TIME: '288.900', BASE_REAL_PEAK: '32975', REAL_TIME_DIFF: '-7.500', REAL_TIME_PCT: '-2.6', REAL_PEAK_DIFF: '5', REAL_PEAK_PCT: '0.0', PR_GROWTH_HEAPS: '18756/26966/34748/43896/50431', diff --git a/tooling/ethrex-block-converter/.gitignore b/tooling/ethrex-block-converter/.gitignore index 6d50c2d01..d4c9a8fb7 100644 --- a/tooling/ethrex-block-converter/.gitignore +++ b/tooling/ethrex-block-converter/.gitignore @@ -1,4 +1,6 @@ /target -# Downloaded ethrex-replay cache JSONs (~1.5 MB each), fetched on demand by -# `make ethrex-real-block-fixture`. +# Downloaded ethrex-replay cache JSONs (~1.5 MB each), plus the rev stamp that +# invalidates them. Fetched on demand by `make ethrex-real-block-cache` (the +# benchmark block, for `regen-real-block-fixture`) and by +# `make test-ethrex-real-block-converter` (this crate's own pinned test block). /caches/ diff --git a/tooling/ethrex-block-converter/Cargo.toml b/tooling/ethrex-block-converter/Cargo.toml index b1796fa60..f61e68a68 100644 --- a/tooling/ethrex-block-converter/Cargo.toml +++ b/tooling/ethrex-block-converter/Cargo.toml @@ -35,11 +35,12 @@ serde_json = "1" # Not used by this crate's code directly — declared to pin k256's feature set. # `lambda-vm-ethrex-crypto` (a dev-dependency) is otherwise the only thing asking # for `expose-field`, and under resolver 3 dev-dependency features are not unified -# into builds that exclude dev-dependencies: `cargo run` (the -# `make ethrex-real-block-fixture` step) and the `cargo test` that follows it in -# CI would resolve two different k256 feature sets, recompiling k256 and every -# ethrex crate above it in the same job. Declaring the features here puts them in -# both resolutions. Keep in sync with crypto/ethrex-crypto's k256 line; verify: +# into builds that exclude dev-dependencies. This crate is built both ways against +# one target dir: `cargo run` for `make regen-real-block-fixture`, `cargo test` for +# `make test-ethrex-real-block-converter`. Without this line the two resolve +# different k256 feature sets, so alternating between them recompiles k256 and every +# ethrex crate above it. Declaring the features here puts them in both resolutions. +# Keep in sync with crypto/ethrex-crypto's k256 line; verify: # cargo tree -e features,no-dev -i k256 # must list expose-field # cargo tree -e features -i k256 # must match the above k256 = { version = "=0.13.4", default-features = false, features = ["arithmetic", "expose-field"] } diff --git a/tooling/ethrex-block-converter/README.md b/tooling/ethrex-block-converter/README.md index 78dc4fd2e..17cf8a2f8 100644 --- a/tooling/ethrex-block-converter/README.md +++ b/tooling/ethrex-block-converter/README.md @@ -20,14 +20,21 @@ All are measured. | serialized size | 32,766 B | 1,110,156 B | | cycles | 8,734,622 | **50,781,557** | | keccak / ecsm calls | 411 / 80 | **10,478** / 116 | -| keccaks per ecrecover | 5.1 | **90** | +| keccaks per ECSM call | 5.1 | **90** | + +The last row counts keccaks per **ECSM call**, which is the ratio of the two rows +above it. One ecrecover issues four ECSM ecalls — `lincomb2_with_oracle` in +`crypto/ethrex-crypto/src/lib.rs` makes four oracle queries, each an `ecsm_mul` — so +per *ecrecover* the figures are 20.6 and 361, and the 80 and 116 above are 4x the 20 +synthetic and 29 real signature checks. Note the real block uses ~5.8x the gas and costs ~5.8x the cycles here, and that it inverts the crypto mix: the synthetic block is ecrecover-bound, the real one keccak- and trie-bound. That inversion is the point — a prover change can move the two numbers in opposite directions. (The gas and cycle ratios agreeing is a coincidence -of this pair, not a rule: cycles/gas is 20.9 here and ranges ~21–38 across the -candidate blocks.) +of this pair, not a rule: cycles/gas is 20.9 for this block on this ELF, and spans +29.7–38.2 across the three candidate blocks when all three are measured on one +common pre-LTO ELF — a ~29% spread.) **Pin the ELF whenever you quote a cycle count.** Counts above were measured on this branch at merge `fdb92f67` (main @ `9ccdaf2`), guest built with **clang 21.1.8**. @@ -140,28 +147,47 @@ it — see [Measured cost of candidate blocks](#measured-cost-of-candidate-block | Where | How to run it | Cost (current default) | |---|---|---| -| `benchmark-pr.yml` | **`/bench`** on a PR — the only workload it proves; also push to main and `workflow_dispatch` | 3 runs, ~4–5 min each (first run at epoch 2^22 confirms) | -| `scripts/bench_verify.sh` | `WORKLOAD=real scripts/bench_verify.sh ` | ~4.5–4.8 min per side, then cached | -| `scripts/perf_diff.sh` | `WORKLOAD=real scripts/perf_diff.sh ` | 5 recordings, so ~25 min | -| `benchmark-gpu.yml` | `/bench-gpu` on a PR — **the default there** | 59.87 s/prove on an RTX 5090 (see below) | -| `scripts/bench_abba.sh` | `WORKLOAD=real scripts/bench_abba.sh ` | ~4 h at 20 pairs — **manual only** | +| `benchmark-pr.yml` | **`/bench`** on a PR — also push to main and `workflow_dispatch` | 3 runs, 158.8 s each (~8 min of proving) | +| `bench-abba.yml` | **`/bench-abba [N]`** on a PR | ~72 min at the default 12 pairs (see below) | +| `benchmark-gpu.yml` | **`/bench-gpu [N]`** on a PR | 59.87 s/prove on an RTX 5090 (see below) | +| `scripts/bench_verify.sh` | `scripts/bench_verify.sh ` | ~2.6 min per side, then cached | +| `scripts/perf_diff.sh` | `scripts/perf_diff.sh ` | 5 recordings, so ~13 min of proving | +| `scripts/bench_abba.sh` | `scripts/bench_abba.sh [base] [pairs]` | 2 x 158.8 s per pair | + +This block is what all three scripts prove by default (`WORKLOAD=real`); pass +`WORKLOAD=synthetic` for the N-plain-transfer fixture instead. `/bench-verify` is the +one flow that pins `synthetic`, because it reports a monolithic arm as well as a +continuation one and a real block does not fit monolithically. None of them hardcode the fixture path or a block number — they read the path from -`make -s print-real-block-fixture` and run `make ethrex-real-block-fixture` when -the `.bin` is absent. - -**`/bench-abba` is deliberately NOT wired to the real block.** The script supports -it (`WORKLOAD=real`, which is what the GPU workflow drives), but the CPU ABBA -workflow still defaults to the synthetic fixture: 20 pairs × 2 proves × ~4.5–4.8 min -is **~3 hours** on the one shared bench runner, which every other bench queues behind. -Run it by hand when a paired test on a real workload is worth that; the option is -here rather than a footgun on a comment trigger. - -**Both bench flows prove this block.** `/bench` runs it sampled on the shared -runner, against the cached baseline main publishes; `/bench-gpu [N]` runs it as -N A/B/B/A pairs on a rented box, comparing PR vs main on the same machine — -absolute GPU times are host-CPU-dependent, so only same-box deltas are -meaningful. +`make -s print-real-block-fixture` and run `make ethrex-real-block-fixture` on every +invocation, so the digest is re-checked rather than trusted. + +**Every bench flow proves this block.** `/bench` runs it sampled on the shared +runner, against the cached baseline main publishes; `/bench-abba [N]` runs it as +N A/B/B/A pairs on that same runner; `/bench-gpu [N]` runs the same pairs on a +rented box, comparing PR vs main on the same machine — absolute GPU times are +host-CPU-dependent, so only same-box deltas are meaningful. + +**Escalating from `/bench` to `/bench-abba`.** `/bench` resolves about 3%: three +runs of a 158.8 s prove, so it reports 3–10% as unresolved rather than as a +verdict. The paired test resolves a 95% delta of `t* × sd / sqrt(N)`, where `sd` +is the pair-delta standard deviation on the runner. **That sd is not measured +yet.** The columns below bracket it between 1.0% — the GPU box's measured 0.64% +pair sd plus margin — and 2.0%, which is `sqrt(2) ×` this runner's measured 1.43% +single-run CV: + +| pairs | wall | resolves (sd 2.0%) | resolves (sd 1.0%) | +|---|---|---|---| +| 8 | ~50 min | 1.7% | 0.8% | +| **12** | **~72 min** | **1.3%** | **0.6%** | +| 20 | ~1h55m | 0.9% | 0.5% | +| 32 | ~3h | 0.7% | 0.4% | + +12 pairs is the default. Wall is two 158.8 s proves per pair plus ~8 min of +setup. The first real `/bench-abba` run **measures** the sd — it is the `sd` +field of the paired-t line in the result comment — and this table should be +re-pinned to that value once it exists. **GPU baseline (measured), and why the GPU epoch is 2^22.** On an RTX 5090 (32,607 MiB) against main @ `9ccdaf2`, same fixture and CLI, one prove per setting: @@ -179,31 +205,36 @@ this one. `benchmark-gpu.yml` defaults the real-block path to 2^22 for this reas raw traces are in `~/workspace/lambda_vm_bench_cache/gpu_epoch_calib_2026-07-31/` (`PROVENANCE.txt`). -That default is **GPU-path only**. `bench_abba.sh` still defaults to 2^20 for the CPU -`/bench-abba`, and the CLI's own `DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2` is still 20: -2^22 needs ~28 GiB of *host* memory on a CPU build, which would break laptops. VRAM -binds on one path and host RAM on the other, so they are not the same question and the -GPU number must not be copied across. **The CPU-server epoch recommendation is pending -calibration.** +2^22 is also what the CPU runner uses, but the two arrive there for different reasons +and must not be derived from each other: VRAM binds on the GPU path and host RAM on the +CPU one. The workflows pin it on both sides (`GPU_REAL_EPOCH_LOG2` here, +`REAL_BLOCK_EPOCH_LOG2` in `benchmark-pr.yml`, `ABBA_REAL_EPOCH_LOG2` in +`bench-abba.yml`); `bench_abba.sh` and `bench_verify.sh` still *default* to 2^20, as +does the CLI's `DEFAULT_CONTINUATION_EPOCH_SIZE_LOG2`, because 2^22 needs ~32 GiB of +host memory on a CPU build (peak RSS on the calibration box) and would break laptops. +See [Choosing the epoch size](#choosing-the-epoch-size) for the CPU tiers. -The CPU bench runner is roughly **4.5–4.8x** the GPU wall time for the same block. +The CPU bench runner is roughly **2.65x** the GPU wall time for the same block: 158.8 s +median against the calibration RTX 5090's 59.87 s, both at epoch 2^22. -Do not derive one from the other in general: the CPU rate (5.31–5.62 s/Mcycle) does not -transfer to the GPU, and the RTX 5090 sweep found the prover CPU-bound at the serial -producer above epoch 2^21, so GPU time lands closer to CPU time than a naive -device-throughput estimate suggests. +Do not derive one from the other in general: the CPU rate (3.13 s/Mcycle on this +block) does not transfer to the GPU, and the RTX 5090 sweep found the prover +CPU-bound at the serial producer above epoch 2^21, so GPU time lands closer to CPU +time than a naive device-throughput estimate suggests. **Continuations are mandatory, not a tuning choice.** Peak heap on a monolithic prove grows ~4.9 GB per million cycles on this workload family (measured on the bench server: `10,728 MB + 2,007 MB/transfer`, R² = 0.998 across 4→20 transfers), so this block would need **~240 GB** monolithically — and a heavier candidate far more. `--continuations` makes peak heap a function of the epoch size instead of the -trace length, so the same block fits in **~32 GiB** at epoch 2^22, the setting both the -CPU bench runner and the GPU path now use — picked by host RAM on one and by VRAM on -the other. See [Choosing the epoch size](#choosing-the-epoch-size) for the full curve -and the other tiers. The -bundle on disk is ~1.9 GB; note that a heavier block pushes it past 2 GiB, which -needs rkyv `pointer_width_64` to serialize. +trace length, so the same block fits in **~32 GiB** at epoch 2^22 on the calibration +box — the setting both the CPU bench runner and the GPU path now use, picked by host +RAM on one and by VRAM on the other. Host peaks are machine-specific: the bench runner +itself measured **~52 GB** of peak heap for the same block and epoch. See +[Choosing the epoch size](#choosing-the-epoch-size) for the full curve and the other +tiers. The bundle on disk is ~1.15 GB (1.12 GB on the GPU path); a block would have to +be ~1.9x heavier to push it past the 2 GiB (2.147 GB) rkyv offset limit, which needs +`pointer_width_64` to serialize. **`/bench` proves this block and nothing else** — 3 sampled runs against the cached 3-run baseline main publishes on every push. `/bench N` changes the sample count @@ -216,20 +247,22 @@ make twice: 1. Its only unique coverage was the **monolithic** prove path, which is vestigial — reportedly slower than a single-epoch continuation. Spending runner time to cover a path we intend to delete is not a trade worth making. -2. Its crypto mix is one **no real block has**: 8.83 ECSM per Mcycle, against a real - block's ~1.3–1.6. Screening against it tunes the prover for a worst case that +2. Its crypto mix is one **no real block has**: 9.16 ECSM per Mcycle, against a real + block's 2.28. Screening against it tunes the prover for a worst case that cannot occur. The synthetic fixtures themselves are not gone — `/bench-growth` still sweeps them for a heap-vs-block-size slope, which needs a family of blocks and so cannot come from one -real one. `/bench-abba` and `/bench-verify` are untouched. +real one, and `/bench-verify` still proves the 20-transfer block so it can report a +monolithic arm as well as a continuation one. **The cost is a shared resource.** One runner carries every `/bench`, `/bench-abba` -and `/bench-verify` in the repo, and a `/bench` now occupies it for **~15–20 min**, on -every comment and every push to main. `BENCH_RUNS_REAL` in `benchmark-pr.yml` is the -dial. Its per-run time on that runner is still unmeasured — the epoch calibration ran -on a different box — so the first run is what tells you whether 3 was the right -number; above ~6 min a run, turn it down. +and `/bench-verify` in the repo, and a `/bench` now occupies it for **~15 min**, on +every comment and every push to main — ~8 min of that is the three proves, the rest is +checkout, a two-sided build, the fixture fetch and the guest ELF. `BENCH_RUNS_REAL` in +`benchmark-pr.yml` is the dial. A run measured **158.8 s** on that runner (median of 3, +2.8% spread, 13 epochs, epoch 2^22), so 3 runs is the right count; the dial is there +for a future block or prover change that takes a run past ~6 min. Cycle counts here (8.73M synthetic, 50.78M real) are from merge `fdb92f67` (main @ `9ccdaf2`, clang 21). They move with guest optimisation and ~2% with the clang major, @@ -313,13 +346,13 @@ repointed ~1 MB fixture cannot become committable by accident. Cost is a property of the block, so it changes with the repoint. All figures are measured, never derived from gas — **cycles per gas is not constant** (20.9 for the -current default), so ranking candidates by gas mispredicts cost. +current default), so sizing a candidate from its gas mispredicts cost. **Current default — main-vintage (merge `fdb92f67`, main @ `9ccdaf2`):** | block | gas | cycles | GPU prove (RTX 5090) | CPU prove | proof | fixture | |---|---|---|---|---|---|---| -| **mainnet 25368371** | 2.43M | **50,781,557** (clang 21)
50,713,534 (clang 18) | **59.87 s** @ epoch 2^22 | ~4.5–4.8x the GPU wall | ~1.9 GB | 1,110,156 B, `61eba49b…` | +| **mainnet 25368371** | 2.43M | **50,781,557** (clang 21)
50,713,534 (clang 18) | **59.87 s** @ epoch 2^22 | **158.8 s** @ epoch 2^22 (2.65x the GPU wall) | 1.15 GB CPU / 1.12 GB GPU | 1,110,156 B, `61eba49b…` | Epoch 2^22 is the GPU recommendation: VRAM binds, 2^22 leaves 28.9% headroom on a 32 GiB card and 2^23 does not fit one at all. See @@ -340,10 +373,13 @@ All three clear the usability screen. Add a row rather than editing the wiring, say which ELF a number came from. Two things these numbers show that a gas-based estimate would have got wrong, and both -survive the vintage change because they are same-ELF comparisons. **Gas does not order -cost:** 25453112 carries *more* gas than hoodi 1265656 yet costs ~25% fewer cycles. -And **fixture size does not track cost** either: the current default is the cheapest -block and the middle-sized fixture. +survive the vintage change because they are same-ELF comparisons. **Gas does not size +cost:** on one common pre-LTO ELF the three blocks run at 30.8, 29.7 +and 38.2 cycles per gas, so budgeting a candidate from gas alone is off by up to ~29% +— 25453112 and hoodi 1265656 sit within 4% of each other on gas (4.24M vs 4.40M) yet +25453112 costs ~25% fewer cycles (125.9M vs 168.3M). Gas happens to *order* these +three correctly; it does not size them. And **fixture size does not track cost** +either: the current default is the cheapest block and the middle-sized fixture. The default is the cheapest of the three, which matters because the CPU workload sits on a single shared bench runner. It was also the only block in a 90-day Dune sweep @@ -359,7 +395,7 @@ machine, not of the block**. Three tiers, all measured: | where | epoch | why | |---|---|---| | GPU, 32 GiB card | **2^22** | VRAM-bound — 2^23 does not fit | -| CPU bench runner (≥64 GiB) | **2^22** | host-RAM-bound — 2^23's 60 GiB does not fit safely | +| CPU bench runner (≥64 GiB) | **2^22** | host-RAM-bound — it already peaks at ~52 GB here, and 2^23 measured 60 GiB on a roomier box | | CPU server, 128 GiB class | **2^23** | the knee; 2^24 fits but is not worth it | | laptops (CLI default) | **2^20** | unchanged, so a plain `cli prove` still works | @@ -385,10 +421,12 @@ A main-vintage anchor also ran on the same box: 2^23 = 344.12 s / 58.87 GiB, i.e faster than branch vintage at the same memory — as expected, since #861 cut cycles and peak RSS is set by the epoch size rather than the trace length. -**Do not read absolute seconds off this table for another machine.** The calibration box -runs ~8.6 s/Mcycle against the bench runner's 5.31–5.62, so the *ratios* between epochs -transfer and the wall times do not. `/bench`'s real-block arm now runs at 2^22 (it was -2^21); the first run after that change establishes the runner's real number. +**Do not read absolute seconds or peak memory off this table for another machine.** This +box took 397.88 s at 2^22 on its branch-vintage ELF; the bench runner takes 158.8 s at +the same epoch on the main-vintage one — a 2.5x gap, against the ~6% the two cycle +counts differ by. The *ratios* between epochs transfer; the wall times do not. Memory +transfers no better: the bench runner peaks at ~52 GB of heap at 2^22 where this box +measured 32.21 GiB RSS. ### Verifying a repoint diff --git a/tooling/ethrex-tests/tests/ethrex.rs b/tooling/ethrex-tests/tests/ethrex.rs index eaa811bf4..6374f2f43 100644 --- a/tooling/ethrex-tests/tests/ethrex.rs +++ b/tooling/ethrex-tests/tests/ethrex.rs @@ -89,14 +89,15 @@ fn test_ethrex_empty_block() { const REAL_BLOCK_FIXTURE: &str = "ethrex_mainnet_25368371.bin"; -/// Host-only acceptance gate for the real-block fixture produced by -/// `tooling/ethrex-block-converter` (`make ethrex-real-block-fixture`): the block is +/// Host-only acceptance gate for the real-block fixture that +/// `make ethrex-real-block-fixture` fetches and checksums (produced offline by +/// `tooling/ethrex-block-converter`): the block is /// re-executed statelessly against its own witness, so a successful run means /// the recovered tries, codes and headers reproduce the header's post-state /// root. Needs no guest ELF, which is what keeps it runnable where the RV64 /// toolchain isn't available. /// -/// Checks the *serialized artifact* specifically — that the committed rkyv +/// Checks the *serialized artifact* specifically — that the published rkyv /// bytes deserialize and execute — which is why it reads the `.bin` rather /// than converting the cache itself. ///