From 5b98a5982d17bebfa5c17194cf1d982a38dc2f1c Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:11:09 +0000 Subject: [PATCH] fix(prism): stop advertising observed tokens as recipe budget Site window tokenBudget was the max telemetry x (~2.6B from a leader run), which the frontend labeled as an egalitarian single-pass window. Recipe 1.2 publishes train_rows/time/step caps, not a fixed token quota. Document train_rows vs full-shard streaming for miners. --- crates/site-api/src/handlers.rs | 2 +- crates/site-data/src/map.rs | 30 +++++++++++++++++++----------- crates/site-types/src/types.rs | 5 ++++- docs/PRISM_RECIPE.md | 24 ++++++++++++++++++++++++ docs/SITE_API.md | 6 +++++- docs/external-miner/prism.md | 9 +++++++++ 6 files changed, 62 insertions(+), 14 deletions(-) diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 753a6b622..63639e062 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -1175,7 +1175,7 @@ mod tests { assert_eq!(v["series"][0]["points"].as_array().unwrap().len(), 2); assert_eq!(v["series"][0]["params"], 12.0); assert_eq!(v["series"][0]["submissionId"], "x2"); - assert_eq!(v["tokenBudget"], 2); + assert_eq!(v["tokenBudget"], 0); assert_eq!(v["series"][1]["points"].as_array().unwrap().len(), 1); assert_eq!(v["series"][1]["points"][0]["step"], 2); } diff --git a/crates/site-data/src/map.rs b/crates/site-data/src/map.rs index b72d1cd30..238f5f75f 100644 --- a/crates/site-data/src/map.rs +++ b/crates/site-data/src/map.rs @@ -580,8 +580,10 @@ pub fn prism_telemetry(detail: &Value) -> Option { } /// Chart x-value for one telemetry point: prefer harness `layer_stats.tokens` -/// (tokens seen) so the window plots against the egalitarian token axis; -/// fall back to the optimizer step when tokens were not reported. +/// (miner-reported tokens seen during that run) so curves plot on a shared +/// observed-token axis; fall back to the optimizer step when tokens were not +/// reported. This is **not** a recipe-published token budget — recipe v1.2 +/// egalitarianism is the pinned shard + seed + wall/step/param caps. fn telemetry_x(point: &PrismTelemetryPoint) -> u32 { let tokens = point .layer_stats @@ -690,15 +692,19 @@ pub fn prism_window( s }) .collect(); - // Axis span = max tokens (or steps) observed across curves so lossPath can - // draw; single-point historical fallbacks (step 0) move to the right edge. - let token_budget = series + // Chart axis span = max tokens (or steps) *observed* across curves so + // lossPath can draw; single-point historical fallbacks (step 0) move to + // the right edge. Do **not** publish that span as `token_budget` — the + // recipe does not fix a token quota (miners stream the pinned shard under + // the 6h / 20k-step caps), and surfacing a leader's ~2.6B tokens as an + // "egalitarian window" misled miners vs `GET /v1/recipe` (`train_rows`). + let axis_span = series .iter() .flat_map(|s| s.points.iter().map(|p| u64::from(p.step))) .max() .unwrap_or(0); - if token_budget > 0 { - let end = u32::try_from(token_budget).unwrap_or(u32::MAX); + if axis_span > 0 { + let end = u32::try_from(axis_span).unwrap_or(u32::MAX); for s in &mut series { if s.points.len() == 1 && s.points[0].step == 0 { s.points[0].step = end; @@ -708,7 +714,8 @@ pub fn prism_window( PrismWindow { dataset, revision, - token_budget, + // Recipe v1.2 publishes row/time/step caps, not a fixed token budget. + token_budget: 0, offset: "pinned".into(), rules_gate: RulesGate { provider, @@ -1112,8 +1119,8 @@ mod tests { assert!((w.series[0].params - 12.0).abs() < f64::EPSILON); assert_eq!(w.series[1].points.len(), 1); assert!((w.series[1].params - 0.0).abs() < f64::EPSILON); - // Budget follows the max x across curves; single-point fallback sits at end. - assert_eq!(w.token_budget, 2); + // Recipe has no fixed token budget; axis remaps single-point fallback. + assert_eq!(w.token_budget, 0); assert_eq!(w.series[1].points[0].step, 2); } @@ -1159,7 +1166,8 @@ mod tests { ); let w = prism_window(Some(&recipe), None, &subs, &telemetry); assert_eq!(w.param_ceiling, 350); - assert_eq!(w.token_budget, 2_000_000); + // Observed tokens appear on the curve axis; they are not a recipe budget. + assert_eq!(w.token_budget, 0); assert_eq!(w.series[0].points[0].step, 100_000); assert_eq!(w.series[0].points[1].step, 2_000_000); assert!((w.series[0].params - 24.0).abs() < f64::EPSILON); diff --git a/crates/site-types/src/types.rs b/crates/site-types/src/types.rs index db5ab6607..444ffce8a 100644 --- a/crates/site-types/src/types.rs +++ b/crates/site-types/src/types.rs @@ -257,7 +257,10 @@ pub struct PrismWindow { pub dataset: String, /// Recipe revision / pin short. pub revision: String, - /// Token budget when known; 0 if the recipe does not publish one. + /// Fixed token budget when the recipe publishes one; **0** for prism + /// recipe ≥1.2 (egalitarian caps are wall-clock / steps / params + pinned + /// shard — not a fixed token quota). Chart axis span is derived from + /// series points, not this field. pub token_budget: u64, /// Always pinned. pub offset: String, diff --git a/docs/PRISM_RECIPE.md b/docs/PRISM_RECIPE.md index 0e1e22951..863ce22ef 100644 --- a/docs/PRISM_RECIPE.md +++ b/docs/PRISM_RECIPE.md @@ -74,6 +74,30 @@ score. | Hard step cap | 20 000 (config may only lower) | | Source size | 128 KiB per script | | Model parameters | ≤ **350 000 000** after `build_model` (`MAX_PARAMS`) | +| `train_rows` (descriptor) | **2048** — baseline / default cut advertised on `GET /v1/recipe` | +| `val_rows` | **256** — frozen val cut scored by the harness (not miner-chosen) | + +### What `train_rows` means (and what it does not) + +`train_rows: 2048` is the **baseline cut** and the value injected into +`ctx["train_rows"]`. The sealed baseline (`training.py`) reads that many texts +from the pinned parquet (~2M GPT-2 tokens for that slice — **not** billions). + +Egalitarian constraints are the **pinned shard + seed + wall/step/param caps**. +The harness hands miners `ctx["dataset_path"]` to the **full** verified +parquet; competitive `training.py` may stream or multi-pass that shard until +the 6h / 20k-step guard fires. Token throughput therefore depends on the miner +loop and the rented GPU — a ~6h RTX 5090 run can report on the order of +**~2.6B** tokens in telemetry. That figure is **observed throughput**, not a +recipe-published “2.6B token window.” + +Do not treat the marketing site’s loss-chart axis (or a leader’s telemetry +peak) as the recipe contract — always trust `GET /v1/recipe` + this doc. + +**Harness note (follow-up, do not hot-fix mid-flight):** `METRICS_JSON.tokens_seen` +currently echoes `TRAIN_ROWS` (2048) even when telemetry `layer_stats.tokens` +shows billions. Changing that field would alter the recipe pin (harness bytes +are hashed) — coordinate a version bump if/when fixing it. ## Recipe pin diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 48a0173e3..866f903c8 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -30,7 +30,11 @@ Leaderboard `elo` is the design `rating` field. When the current round has no winners yet (`ratings: []`), `/v1/site/arenas/design/leaderboard` surfaces the previous round's standings (`roundId` = previous) rather than an empty board. Prism window series use real terminal `bpb` with a single `[final]` point when -no step curve is stored. +no step curve is stored. `PrismWindow.tokenBudget` is **0** unless a recipe +publishes a fixed token quota (prism ≥1.2 does not — caps are wall-clock / +steps / params). Chart x-values still come from miner telemetry +(`layer_stats.tokens` when present); clients must not label the max observed +x as an egalitarian “token window.” `GET /v1/site/arenas/{slug}/submissions` and `/leaderboard` accept optional `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle, diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index d6c37ed8b..ceb8f5d44 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -62,6 +62,15 @@ curl -sS "$BASE_GATEWAY/challenge/prism/v1/recipe" curl -sS "$BASE_GATEWAY/challenge/prism/v1/recipe/baseline" ``` +Live production (recipe **1.2.0**) advertises `train_rows: 2048`, +`val_rows: 256`, `train_hours_cap: 6.0`, `max_train_steps: 20000`, +`max_params: 350000000`, and `pin_hex` (sha over version + caps + dataset + +harness). The sealed baseline only trains on the 2048-row cut (~2M GPT-2 +tokens) and scores poorly by design; competitive entries may stream the full +pinned FineWeb-Edu shard for up to 6h. Site chart labels that show “~2.6B +tokens · single pass” were **observed leader telemetry**, not a fixed recipe +quota — trust `/v1/recipe`, not the chart meta line. + `POST /v1/submissions` is idempotent by `submission_id`. ## Submission gating (1-max)