From ef56732aee82f0203fad95949d2ad660433923d5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:48:01 +0000 Subject: [PATCH 1/7] fix(prism): restore WTA emission, pre-pod screens, 8 concurrent evals Collapse competition credits to a single Score leaf so Prism's 50% share goes to one winner; fail static/similarity cheats before Lium rent; scale orchestrator workers to 8 on prod. --- bins/prism-challenge/src/main.rs | 2 +- crates/challenge-agentic/src/lib.rs | 2 + crates/challenge-agentic/src/sim.rs | 11 +- crates/challenge-agentic/src/static_checks.rs | 94 +++++++++++++ crates/prism-challenge/src/orchestrator.rs | 129 +++++++++++++----- .../prism-challenge/tests/cheat_arch_copy.rs | 8 +- crates/prism-challenge/tests/cheat_metrics.rs | 11 +- crates/prism-challenge/tests/copy_gate.rs | 12 +- crates/prism-emit/src/lib.rs | 13 +- crates/prism-emit/tests/epoch_semantics.rs | 14 +- crates/prism-pipeline/src/config.rs | 6 +- crates/prism-registry/src/competition.rs | 58 +++++++- crates/prism-registry/src/lib.rs | 2 +- deploy/compose/env-prod.yml | 2 + deploy/env/prism-challenge.env.example | 2 +- docker-compose.yml | 2 +- docs/COMPLETENESS.md | 2 +- docs/PRISM.md | 62 +++++---- docs/evidence/prism-wta-2026-08-08.md | 36 +++++ docs/external-miner/prism.md | 12 +- .../prism-enable-lium-and-emission.md | 3 +- 21 files changed, 378 insertions(+), 105 deletions(-) create mode 100644 crates/challenge-agentic/src/static_checks.rs create mode 100644 docs/evidence/prism-wta-2026-08-08.md diff --git a/bins/prism-challenge/src/main.rs b/bins/prism-challenge/src/main.rs index 0f084d5bb..13618ee31 100644 --- a/bins/prism-challenge/src/main.rs +++ b/bins/prism-challenge/src/main.rs @@ -70,7 +70,7 @@ struct Cli { #[arg( long, env = "PRISM_MAX_CONCURRENT_EVALS", - default_value_t = 1, + default_value_t = 8, global = true )] max_concurrent_evals: u32, diff --git a/crates/challenge-agentic/src/lib.rs b/crates/challenge-agentic/src/lib.rs index 495cccbfb..19bf1db6e 100644 --- a/crates/challenge-agentic/src/lib.rs +++ b/crates/challenge-agentic/src/lib.rs @@ -18,6 +18,7 @@ mod agent; mod llm; mod prompts; mod sim; +mod static_checks; mod tools; mod types; @@ -26,6 +27,7 @@ pub use challenge_ast::{copy_gate, CopyGateHit, GateCorpusEntry}; pub use llm::{load_api_key_file, DEFAULT_MODEL}; pub use prompts::{AGENTIC_PROMPT_VERSION, DESIGN_DOMAIN_RULES, PRISM_DOMAIN_RULES}; pub use sim::{SimAgent, SIM_CHEAT_BPS, SIM_SUSPICIOUS_BPS}; +pub use static_checks::{static_source_cheat, training_has_telemetry_hooks, StaticCheatHit}; pub use types::{ AgenticBackend, AgenticError, AgenticVerdict, CheatCode, ContainerReviewRequest, CorpusEntry, ReviewRequest, VerdictKind, OPENROUTER_API_BASE, diff --git a/crates/challenge-agentic/src/sim.rs b/crates/challenge-agentic/src/sim.rs index fa4222e6d..9255c7d4e 100644 --- a/crates/challenge-agentic/src/sim.rs +++ b/crates/challenge-agentic/src/sim.rs @@ -79,6 +79,10 @@ impl AgenticBackend for SimAgent { return Ok(v); } + // Source-only screens are also available via [`crate::static_source_cheat`] + // for the pre-pod orchestrator path; sim keeps the in-review copies so + // metrics-relative checks and corpus AST still share one backend. + if let Some(v) = pages_scrape_cheat_verdict(req)? { return Ok(v); } @@ -187,12 +191,7 @@ fn telemetry_hooks_verdict( return None; } let (path, src) = primaries.iter().find(|(p, _)| p.ends_with("training.py"))?; - let imports_shim = src.contains("prism_telemetry") - || src.contains("ctx[\"telemetry\"]") - || src.contains("ctx['telemetry']"); - let calls_report = src.contains(".report("); - let calls_finish = src.contains("finish_evaluation("); - if imports_shim && calls_report && calls_finish { + if crate::training_has_telemetry_hooks(src) { return None; } Some(AgenticVerdict { diff --git a/crates/challenge-agentic/src/static_checks.rs b/crates/challenge-agentic/src/static_checks.rs new file mode 100644 index 000000000..6684bc0f8 --- /dev/null +++ b/crates/challenge-agentic/src/static_checks.rs @@ -0,0 +1,94 @@ +//! Cheap source-only cheat screens (no GPU, no private eval assets). +//! +//! Run these **before** renting a Lium pod so a bad submission fails fast +//! instead of burning hours of GPU. Metrics/receipt consistency checks stay +//! post-eval (they need harness output). + +use crate::types::CheatCode; + +/// One static source finding. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StaticCheatHit { + /// Cheat taxonomy code. + pub code: CheatCode, + /// Human-readable reason (safe to surface in error_detail). + pub rationale: String, +} + +/// Scan miner sources for cheap, deterministic cheat patterns. +/// +/// Order: hardcoded `METRICS_JSON=` short-circuit first, then missing Prism +/// telemetry hooks in `training.py`. Returns the first hit. +#[must_use] +pub fn static_source_cheat( + architecture_py: &str, + training_py: &str, +) -> Option { + for (path, src) in [ + ("architecture.py", architecture_py), + ("training.py", training_py), + ] { + if src.contains("METRICS_JSON=") { + return Some(StaticCheatHit { + code: CheatCode::EvalShortCircuit, + rationale: format!("static: hardcoded METRICS_JSON in {path}"), + }); + } + } + if !training_has_telemetry_hooks(training_py) { + return Some(StaticCheatHit { + code: CheatCode::MissingTelemetryHooks, + rationale: "static: training.py missing prism_telemetry report/finish_evaluation hooks" + .into(), + }); + } + None +} + +/// Prism telemetry-hook contract (recipe ≥ 1.1.0). +#[must_use] +pub fn training_has_telemetry_hooks(training_py: &str) -> bool { + let imports_shim = training_py.contains("prism_telemetry") + || training_py.contains("ctx[\"telemetry\"]") + || training_py.contains("ctx['telemetry']"); + let calls_report = training_py.contains(".report("); + let calls_finish = training_py.contains("finish_evaluation("); + imports_shim && calls_report && calls_finish +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn metrics_json_short_circuit() { + let hit = static_source_cheat( + "def build_model(ctx):\n pass\n", + "def train(m, ctx):\n print('METRICS_JSON={}')\n", + ) + .expect("hit"); + assert_eq!(hit.code, CheatCode::EvalShortCircuit); + } + + #[test] + fn missing_hooks() { + let hit = static_source_cheat( + "def build_model(ctx):\n pass\n", + "def train(m, ctx):\n return {}\n", + ) + .expect("hit"); + assert_eq!(hit.code, CheatCode::MissingTelemetryHooks); + } + + #[test] + fn clean_hooks() { + let train = concat!( + "import prism_telemetry\n", + "def train(m, ctx):\n", + " prism_telemetry.report(loss=1.0, step=1)\n", + " prism_telemetry.finish_evaluation()\n", + " return {}\n", + ); + assert!(static_source_cheat("def build_model(ctx):\n pass\n", train).is_none()); + } +} diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index cce1c9689..02d2d2aa4 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -1,23 +1,27 @@ //! Lium job orchestrator: DB-backed state machine, recovery, epoch emitter. //! -//! Workers claim `queued` rows, rent + run the recipe, run master-side LLM -//! review + cheap similarity + agentic anti-cheat, and compute the +//! Workers claim `queued` rows, run cheap source screens (copy gate, static +//! cheat patterns, AST similarity) **before** renting a Lium pod, then run +//! the recipe + master-side LLM review + agentic anti-cheat, and compute the //! chain-facing score. Leaf emission is decoupled from finalizes: the //! epoch-close emitter ([`prism_emit::EpochEmitter`], driven by //! [`Orchestrator::run_emitter`]) assigns every newly-finalized row to the //! next chain-epoch boundary's D24 set via the emission outbox //! (`emitted_epoch` watermark + emit cursor), so independent same-epoch //! scorers all land and each scoring run is assigned exactly once. Positive -//! scores then carry into later epochs' competition sets until superseded. -//! All state lives in the store, so the API is a pure projection and restarts -//! sweep orphans. +//! scores then carry into later epochs' competition sets until superseded; +//! leaf emission applies WTA so only the single best hotkey gets Prism's +//! share. All state lives in the store, so the API is a pure projection and +//! restarts sweep orphans. use std::sync::Arc; use std::time::Duration; use bundle::NoScoreReasonCode; use chain::ChainClient; -use challenge_agentic::{copy_gate, AgenticBackend, AgenticVerdict, VerdictKind}; +use challenge_agentic::{ + copy_gate, static_source_cheat, AgenticBackend, AgenticVerdict, VerdictKind, +}; use challenge_common::{expected_set_at_chain, PinnedBlockHash}; use crypto::KEY_LEN; use prism_emit::EpochEmitter; @@ -330,12 +334,34 @@ impl Orchestrator { let id = row.id.clone(); info!(submission_id = %id, miner = %row.miner_hotkey, "prism eval start"); - // Phase 0: pre-LLM copy gate on architecture.py (created_at ordered). - // A byte/AST copy of a strictly-earlier architecture is terminal - // `rejected` with Score(0) — no pod time, no LLM spend. + // Phase 0: pre-pod cheap screens (no GPU, no private eval assets). + // Copy gate → static cheat patterns → AST similarity. Fail-fast with + // Score(0) so a bad submission never rents a Lium pod (~6h waste). if self.copy_gate_step(&row).await { return Ok(()); } + if self.static_source_step(&row).await { + return Ok(()); + } + let similarity = match self.similarity_step(&id, &row).await { + Ok(v) => v, + Err(e) => { + if self.maybe_auto_retry(&row, "ast_infra", &e).await { + return Ok(()); + } + self.fail_terminal(&row, "ast_infra", &e).await; + return Ok(()); + } + }; + if matches!( + similarity.kind, + prism_review::SimilarityKind::Copied | prism_review::SimilarityKind::Suspicious + ) { + let detail = format!("pre-pod similarity: {:?}", similarity.kind); + self.reject_pre_pod(&row, Some(similarity), None, detail) + .await; + return Ok(()); + } // Phase 1: provision + recipe exec + terminate (always verified). // Lium/infra failures auto-retry (install class); budget exhaustion is @@ -359,19 +385,8 @@ impl Orchestrator { return Ok(()); }; - // Phase 3: cheap similarity (AST infra → auto-retry, then terminal). - let similarity = match self.similarity_step(&id, &row).await { - Ok(v) => v, - Err(e) => { - if self.maybe_auto_retry(&row, "ast_infra", &e).await { - return Ok(()); - } - self.fail_terminal(&row, "ast_infra", &e).await; - return Ok(()); - } - }; - - // Phase 4: agentic anti-cheat (LLM infra → auto-retry, then terminal). + // Phase 3: agentic anti-cheat (needs metrics/receipt; post-pod). + // Source-only screens already ran pre-pod; this catches metrics forge. let Some(agentic) = self .agentic_step(&id, &row, metrics.as_ref(), receipt.as_ref()) .await @@ -474,6 +489,59 @@ impl Orchestrator { }], prompt_version: prism_review::SIMILARITY_PROMPT_VERSION, }; + self.reject_pre_pod( + row, + Some(similarity), + Some(serde_json::json!({ + "gate": "copy_created_at", + "nearest_id": hit.nearest_id, + "similarity_bps": hit.similarity_bps, + "byte_identical": hit.byte_identical, + })), + format!( + "copy gate: architecture clones {} (bps={})", + hit.nearest_id, hit.similarity_bps + ), + ) + .await; + true + } + + /// Static source cheat screen (METRICS_JSON / telemetry hooks). Pre-pod. + /// Returns `true` when the row was finalized terminal `rejected`. + async fn static_source_step(&self, row: &SubmissionState) -> bool { + let Some(hit) = static_source_cheat(&row.architecture_py, &row.training_py) else { + return false; + }; + warn!( + submission_id = %row.id, + code = ?hit.code, + rationale = %hit.rationale, + "static source cheat rejected (pod skipped)" + ); + self.reject_pre_pod( + row, + None, + Some(serde_json::json!({ + "gate": "static_source", + "cheat_code": format!("{:?}", hit.code), + "rationale": hit.rationale, + })), + hit.rationale.clone(), + ) + .await; + true + } + + /// Terminal Score(0) reject before any Lium rent. Shared by copy gate, + /// static screens, and pre-pod similarity. + async fn reject_pre_pod( + &self, + row: &SubmissionState, + similarity: Option, + detail: Option, + error_detail: String, + ) { let _ = self .store .apply( @@ -481,21 +549,13 @@ impl Orchestrator { &StatePatch { status: Some(Stage::Rejected), final_score: Some(FinalScore::Score(0)), - similarity: Some(similarity), - error_detail: Some(format!( - "copy gate: architecture clones {} (bps={})", - hit.nearest_id, hit.similarity_bps - )), + similarity, + error_detail: Some(error_detail), ..StatePatch::default() }, Some(&StageEvent { stage: Stage::Rejected, - detail: Some(serde_json::json!({ - "gate": "copy_created_at", - "nearest_id": hit.nearest_id, - "similarity_bps": hit.similarity_bps, - "byte_identical": hit.byte_identical, - })), + detail, at_ms: 0, }), ) @@ -510,9 +570,6 @@ impl Orchestrator { ) .await; } - // The Score(0) enters the emission outbox; the epoch-close emitter - // lands it in the next boundary's D24 set. - true } /// Pod phase. Returns `(bpb, receipt)` on full success. diff --git a/crates/prism-challenge/tests/cheat_arch_copy.rs b/crates/prism-challenge/tests/cheat_arch_copy.rs index 8102fe51b..0b25185c3 100644 --- a/crates/prism-challenge/tests/cheat_arch_copy.rs +++ b/crates/prism-challenge/tests/cheat_arch_copy.rs @@ -136,11 +136,13 @@ async fn baseline_arch_train_copy_scores_zero() { assert!(orch.cycle_once().await.unwrap()); let row = store.get(&id).await.unwrap().expect("row"); - assert!( - matches!(row.status, Stage::Terminated | Stage::Failed), - "status={:?}", + assert_eq!( + row.status, + Stage::Rejected, + "baseline arch copy must fail pre-pod similarity, got {:?}", row.status ); + assert!(row.pod_id.is_none(), "arch copy must not rent a pod"); assert_eq!( row.final_score, Some(FinalScore::Score(0)), diff --git a/crates/prism-challenge/tests/cheat_metrics.rs b/crates/prism-challenge/tests/cheat_metrics.rs index 517783fcc..97deaa6c9 100644 --- a/crates/prism-challenge/tests/cheat_metrics.rs +++ b/crates/prism-challenge/tests/cheat_metrics.rs @@ -141,11 +141,16 @@ def train(model, ctx): assert!(orch.cycle_once().await.unwrap()); let row = store.get(&id).await.unwrap().expect("row"); - assert!( - matches!(row.status, Stage::Terminated | Stage::Failed), - "status={:?}", + assert_eq!( + row.status, + Stage::Rejected, + "static METRICS_JSON screen must reject pre-pod, got {:?}", row.status ); + assert!( + row.pod_id.is_none(), + "hardcoded METRICS_JSON must not rent a pod" + ); assert_eq!( row.final_score, Some(FinalScore::Score(0)), diff --git a/crates/prism-challenge/tests/copy_gate.rs b/crates/prism-challenge/tests/copy_gate.rs index 0b4f9fbe9..103cb61a8 100644 --- a/crates/prism-challenge/tests/copy_gate.rs +++ b/crates/prism-challenge/tests/copy_gate.rs @@ -239,9 +239,10 @@ async fn ast_copy_with_renames_is_rejected() { } #[tokio::test] -async fn same_arch_same_timestamp_passes_the_gate() { +async fn same_arch_same_timestamp_passes_copy_gate_then_similarity() { // created_at ties cannot be ordered → the copy gate must NOT reject; - // the row proceeds to the normal pipeline (sim: terminates with a score). + // pre-pod cheap similarity still catches the identical architecture + // (Score(0), no pod) before any Lium rent. let store = Arc::new(MemoryPrismStore::new()); let chain = Arc::new(LockedFake(Mutex::new(fake_chain()))); let orch = Arc::new(mk_orchestrator(&store, &chain)); @@ -275,10 +276,11 @@ async fn same_arch_same_timestamp_passes_the_gate() { .await .unwrap() .expect("row b"); - assert_ne!(b.status, Stage::Rejected, "tie must not hard-reject"); - // The LLM similarity path (SimReviewer, arch-only) still judges the copy. + assert_eq!(b.status, Stage::Rejected, "status={:?}", b.status); + assert!(b.pod_id.is_none(), "tie copy must not rent a pod"); assert_eq!(b.final_score, Some(FinalScore::Score(0))); - assert!(matches!(b.status, Stage::Terminated | Stage::Failed)); + let sim = b.similarity.expect("similarity recorded"); + assert!(matches!(sim.kind, prism_review::SimilarityKind::Copied)); } #[tokio::test] diff --git a/crates/prism-emit/src/lib.rs b/crates/prism-emit/src/lib.rs index 54d25178f..06d12d01d 100644 --- a/crates/prism-emit/src/lib.rs +++ b/crates/prism-emit/src/lib.rs @@ -23,9 +23,10 @@ //! re-enter the outbox (`reset_for_retry` clears the watermark). //! - **Positive scores carry forward**: after outbox assignment, a //! `Score(v>0)` row keeps participating in every later epoch's -//! competition set until a better/valid score supersedes it via `max` -//! (lattice-proportional — not WTA). Empty or reject-only fresh batches -//! therefore do not burn the prism share. +//! competition set until a better/valid score supersedes it via `max`. +//! Leaf emission then applies **WTA** ([`prism_registry::apply_wta`]) so +//! only the single best hotkey receives a positive Score leaf. Empty or +//! reject-only fresh batches therefore do not burn the prism share. //! - Epochs during a master outage carry no *new* outbox rows; the first //! epoch after recovery still includes active positive scores plus any //! backlog (the seal always pins fresh epochs — stale ones can never @@ -207,7 +208,8 @@ pub fn build_epoch_leaves( batch: &[EpochScoreRow], arch_owners: &BTreeMap, ) -> Result, EmitError> { - let by_miner = prism_registry::competition_scores(batch, arch_owners); + let by_miner = + prism_registry::apply_wta(prism_registry::competition_scores(batch, arch_owners)); let mut scores: BTreeMap = BTreeMap::new(); let mut expected_set: BTreeSet = BTreeSet::new(); for p in &expected.participants { @@ -295,9 +297,10 @@ mod tests { ) .unwrap(); assert_eq!(leaves.len(), 3); + // WTA: only the argmax (b=200) keeps a positive Score leaf. assert!(matches!( soa_of(&leaves, &a), - ScoreOrAbsence::Score { value: 100 } + ScoreOrAbsence::Score { value: 0 } )); assert!(matches!( soa_of(&leaves, &b), diff --git a/crates/prism-emit/tests/epoch_semantics.rs b/crates/prism-emit/tests/epoch_semantics.rs index 5770db5a0..3f6b4c372 100644 --- a/crates/prism-emit/tests/epoch_semantics.rs +++ b/crates/prism-emit/tests/epoch_semantics.rs @@ -147,7 +147,8 @@ async fn independent_same_epoch_scorers_both_land() { assert_eq!(s.epoch, 7); assert_eq!(s.leaves, 3, "D24-complete set"); assert_eq!(s.batch, 2, "both scorers in one batch"); - assert_score(&leaf_soa(&s, 0xAA), 100_000); + // WTA: only the argmax (BB=200k) keeps a positive Score leaf. + assert_score(&leaf_soa(&s, 0xAA), 0); assert_score(&leaf_soa(&s, 0xBB), 200_000); assert_not_attempted(&leaf_soa(&s, 0xCC)); assert_eq!(store.emit_cursor(541).await.unwrap(), Some(7)); @@ -227,7 +228,8 @@ async fn no_double_emission_across_epochs() { .unwrap(); let s8 = em.tick(8, &exp).await.unwrap().expect("epoch 8 emits"); assert_eq!(s8.batch, 1, "only the new row is freshly assigned"); - assert_score(&leaf_soa(&s8, 0xAA), 100_000); + // WTA: BB's 900k beats AA's carried 100k — only BB emits positive. + assert_score(&leaf_soa(&s8, 0xAA), 0); assert_score(&leaf_soa(&s8, 0xBB), 900_000); assert_eq!(store.emit_batch(541, 7).await.unwrap().len(), 1); assert_eq!(store.emit_batch(541, 8).await.unwrap().len(), 1); @@ -304,16 +306,16 @@ async fn competition_credits_survive_batching() { assert_not_attempted(&leaf_soa(&s7, 0xBB)); // Challenger trains the published arch and lands in a later epoch: - // owner credit flows to A, own best to B — max lattice, never summed. + // competition credits both to 900k; WTA tie-breaks to AA (lex smaller). let mut chall = scored_row("sub-chall", &hk(0xBB), 7, FinalScore::Score(900_000)); chall.arch_id = Some("arch_0123456789abcdef".into()); store.insert_queued(&chall).await.unwrap(); let s8 = em.tick(8, &exp).await.unwrap().expect("epoch 8"); assert_eq!(s8.batch, 1); assert_score(&leaf_soa(&s8, 0xAA), 900_000); - assert_score(&leaf_soa(&s8, 0xBB), 900_000); + assert_score(&leaf_soa(&s8, 0xBB), 0); - // Same-epoch variant: both rows in one batch → identical credits in one set. + // Same-epoch variant: both rows in one batch → same WTA outcome. let store2 = Arc::new(MemoryPrismStore::new()); store2 .publish_arch(&ArchitectureRecord { @@ -333,7 +335,7 @@ async fn competition_credits_survive_batching() { let s = em2.tick(7, &exp).await.unwrap().expect("emit"); assert_eq!(s.batch, 2); assert_score(&leaf_soa(&s, 0xAA), 900_000); - assert_score(&leaf_soa(&s, 0xBB), 900_000); + assert_score(&leaf_soa(&s, 0xBB), 0); } /// Crash recovery: a batch assigned but never cursor-completed (crashed diff --git a/crates/prism-pipeline/src/config.rs b/crates/prism-pipeline/src/config.rs index 8e0cc4719..b0112b86a 100644 --- a/crates/prism-pipeline/src/config.rs +++ b/crates/prism-pipeline/src/config.rs @@ -9,7 +9,7 @@ pub struct PrismConfig { pub max_price_per_hour: f64, /// Max lifetime hours (≥ 1). pub max_lifetime_hours: f64, - /// Global concurrent Lium evals (v1 = 1). + /// Global concurrent Lium evals (horizontal scale; prod default 8). pub max_concurrent_evals: u32, /// SSH public keys for Real Lium rent (empty for Sim). pub ssh_public_keys: Vec, @@ -37,7 +37,7 @@ impl PrismConfig { require_image_digest: false, // pin when operator supplies digest max_price_per_hour: 2.5, max_lifetime_hours: 2.0, - max_concurrent_evals: 1, + max_concurrent_evals: 8, ssh_public_keys: vec![], default_image_digest: None, preferred_offer_id: None, @@ -53,7 +53,7 @@ impl PrismConfig { require_image_digest: false, max_price_per_hour: 1.5, max_lifetime_hours: 1.0, - max_concurrent_evals: 1, + max_concurrent_evals: 8, ssh_public_keys: vec![], default_image_digest: None, preferred_offer_id: None, diff --git a/crates/prism-registry/src/competition.rs b/crates/prism-registry/src/competition.rs index 614618e75..61e70ed34 100644 --- a/crates/prism-registry/src/competition.rs +++ b/crates/prism-registry/src/competition.rs @@ -12,11 +12,14 @@ //! best epoch result (`max(Score)` over all rows linked to that arch, any //! trainer) is credited to the arch's owner — the owner is rewarded when //! *anyone* trains well on their architecture. -//! - **Emission**: per hotkey `max(own credits, owner credits)` — never +//! - **Per-hotkey credit**: `max(own credits, owner credits)` — never //! summed, so the SCORE_MAX lattice bound and the no-double-count property //! hold by construction. Hotkeys whose rows are all `NoScore` keep their //! absence; `Score(0)` rows (cheat / copy-gate reject) emit 0 and never //! set an arch's best. +//! - **WTA leaf emission**: [`apply_wta`] collapses the credit map to a +//! single positive `Score` (argmax; lexicographically smallest hotkey on +//! ties). Prism's emission share goes to that one winner. use std::collections::BTreeMap; @@ -79,6 +82,35 @@ pub fn competition_scores( out } +/// Winner-take-all collapse: keep only the single highest positive score. +/// +/// Ties break by lexicographically smallest hotkey (stable, hex-encoded). +/// Non-positive `Score(0)` and `NoScore` rows are preserved unchanged; +/// every other positive `Score` is zeroed so the aggregator cannot soft- +/// allocate Prism's share across multiple hotkeys. +#[must_use] +pub fn apply_wta(scores: BTreeMap) -> BTreeMap { + let winner = scores + .iter() + .filter_map(|(hk, s)| match s { + FinalScore::Score(v) if *v > 0 => Some((hk.as_str(), *v)), + _ => None, + }) + // Higher score wins; on equal score prefer the smaller hotkey. + .max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(a.0))) + .map(|(hk, _)| hk.to_owned()); + let Some(winner) = winner else { + return scores; + }; + scores + .into_iter() + .map(|(hk, s)| match &s { + FinalScore::Score(v) if *v > 0 && hk != winner => (hk, FinalScore::Score(0)), + _ => (hk, s), + }) + .collect() +} + #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] @@ -167,4 +199,28 @@ mod tests { // Y's epoch best is 800k (by A) → credited to Y's owner C. assert_eq!(out.get("cc"), Some(&FinalScore::Score(800_000))); } + + #[test] + fn wta_keeps_only_the_argmax_score() { + let rows = vec![ + row("aa", None, 177_155), + row("bb", Some("arch_x"), 111_595), + ]; + let credits = competition_scores(&rows, &owners(&[("arch_x", "cc")])); + // Credits: aa=177155, bb=111595, cc=111595 (owner). + let wta = apply_wta(credits); + assert_eq!(wta.get("aa"), Some(&FinalScore::Score(177_155))); + assert_eq!(wta.get("bb"), Some(&FinalScore::Score(0))); + assert_eq!(wta.get("cc"), Some(&FinalScore::Score(0))); + } + + #[test] + fn wta_tie_breaks_by_lexicographically_smallest_hotkey() { + let mut credits = BTreeMap::new(); + credits.insert("bb".into(), FinalScore::Score(900_000)); + credits.insert("aa".into(), FinalScore::Score(900_000)); + let wta = apply_wta(credits); + assert_eq!(wta.get("aa"), Some(&FinalScore::Score(900_000))); + assert_eq!(wta.get("bb"), Some(&FinalScore::Score(0))); + } } diff --git a/crates/prism-registry/src/lib.rs b/crates/prism-registry/src/lib.rs index fdbcdc56e..faef79edc 100644 --- a/crates/prism-registry/src/lib.rs +++ b/crates/prism-registry/src/lib.rs @@ -20,6 +20,6 @@ mod competition; mod hooks; mod publish; -pub use competition::competition_scores; +pub use competition::{apply_wta, competition_scores}; pub use hooks::post_score_hooks; pub use publish::{TopModelPublisher, TopModelRequest, TOPMODEL_REPO_PATH}; diff --git a/deploy/compose/env-prod.yml b/deploy/compose/env-prod.yml index 9ceb0326d..0f7e32190 100644 --- a/deploy/compose/env-prod.yml +++ b/deploy/compose/env-prod.yml @@ -49,6 +49,8 @@ services: prism-challenge: environment: PRISM_FORCE_SIM: "false" + # Horizontal scale: N orchestrator workers each claim_next under a semaphore. + PRISM_MAX_CONCURRENT_EVALS: "8" # Emitter/gating/epoch-feed chain reads share the gateway failover list. BASE_CHAIN_ENDPOINTS: "wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443" design-challenge: diff --git a/deploy/env/prism-challenge.env.example b/deploy/env/prism-challenge.env.example index 42662e783..15ac74d71 100644 --- a/deploy/env/prism-challenge.env.example +++ b/deploy/env/prism-challenge.env.example @@ -12,7 +12,7 @@ BASE_NETUID=541 # BASE_CHAIN_ENDPOINTS=wss://bittensor-finney.api.onfinality.io/public-ws,wss://entrypoint-finney.opentensor.ai:443 # Orchestrator knobs (defaults in place). -# PRISM_MAX_CONCURRENT_EVALS=1 +# PRISM_MAX_CONCURRENT_EVALS=8 # PRISM_DATASET_SHA256=<256-hex pin override for the fineweb-edu shard> # LLM reviewer: lives by OPENROUTER_API_KEY_FILE mount (see diff --git a/docker-compose.yml b/docker-compose.yml index 41c8dc745..e88037693 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -205,7 +205,7 @@ services: LIUM_SSH_PUBLIC_KEY_FILE: /run/base/lium/ssh_ed25519.pub OPENROUTER_API_KEY_FILE: /run/base/openrouter/api_key BASE_CHALLENGE_GATEWAY_ENDPOINT: ${BASE_CHALLENGE_GATEWAY_ENDPOINT:-http://gateway:8080} - PRISM_MAX_CONCURRENT_EVALS: "${PRISM_MAX_CONCURRENT_EVALS:-1}" + PRISM_MAX_CONCURRENT_EVALS: "${PRISM_MAX_CONCURRENT_EVALS:-8}" # Pods need a while for sshd after RUNNING on the control plane. PRISM_SSH_ATTEMPTS: "${PRISM_SSH_ATTEMPTS:-30}" PRISM_SSH_RETRY_SECS: "${PRISM_SSH_RETRY_SECS:-10}" diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 8d69b91ef..965ca9e2f 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -104,7 +104,7 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | design rating / elimination | done | Integer Elo (K=32), bottom 20% / 4-round cooldown, exact-E leaves. | | design API | done | Harness/quota/runs/viewer/annotate/ops on `:8093`. | | prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. | -| prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), sweeper (7h grace), boot recovery, epoch-close batched D24 leaf emission (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry each epoch, migration 0012). | +| prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (7h grace), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | | prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. | | prism LLM review | done | `prism-review` quality + similarity prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; anti-copy forces `Copied`/`Suspicious` → Score 0. | | prism API | done | Full status surface: submissions list/detail/events/status/jobs/recipe/baseline, idempotent accept. | diff --git a/docs/PRISM.md b/docs/PRISM.md index 10c4db12f..6497a9a30 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -41,11 +41,11 @@ hypertraining B300 tournament code. ```mermaid stateDiagram-v2 [*] --> Queued: POST /v1/submissions - Queued --> Rejected: pre-LLM copy gate (arch copy of earlier submission) - Queued --> Provisioning: worker claims row + Queued --> Rejected: pre-pod screens (copy gate / static cheat / similarity) + Queued --> Provisioning: worker claims + pre-pod screens pass Provisioning --> Running: pod SSH + harness up Running --> Reviewing: METRICS_JSON collected - Reviewing --> AgenticReview: arch-only similarity + quality + Reviewing --> AgenticReview: quality + post-pod agentic AgenticReview --> Scoring: submit_verdict Scoring --> Terminated: finalized row enters the emission outbox Provisioning --> Failed: offer/rent timeout @@ -132,15 +132,18 @@ before submit, the cursor advances only after the full set landed, and a crash mid-submit replays the identical assigned set on the next tick (first-write-wins with identical values converges). After assignment, a positive `Score(v>0)` keeps participating in every later epoch's competition -set until a better/valid score supersedes it via lattice `max` (not WTA) — so -an empty or reject-only fresh batch does not burn the prism share. `Score(0)` -rejects and `NoScore` absences do not carry. A manually retried + re-scored -row re-enters the outbox (`reset_for_retry` clears the watermark); its old -leaf stays immutable history in its original epoch. Epochs during a master -outage carry no *new* outbox rows; the first epoch after recovery still -includes active positive scores plus any backlog (seals always pin fresh -epochs — stale bundles can never Match on-chain). Run **exactly one** -prism-challenge emitter instance per netuid (single master topology). +set until a better/valid score supersedes it via lattice `max` — so an empty +or reject-only fresh batch does not burn the prism share. Leaf emission then +applies **winner-take-all** (`prism_registry::apply_wta`): only the single +highest positive credit (lexicographically smallest hotkey on ties) receives a +positive Score leaf; every other positive credit is zeroed. `Score(0)` rejects +and `NoScore` absences do not carry. A manually retried + re-scored row +re-enters the outbox (`reset_for_retry` clears the watermark); its old leaf +stays immutable history in its original epoch. Epochs during a master outage +carry no *new* outbox rows; the first epoch after recovery still includes +active positive scores plus any backlog (seals always pin fresh epochs — +stale bundles can never Match on-chain). Run **exactly one** prism-challenge +emitter instance per netuid (single master topology). **Competition scoring (epoch-local, SCORE_MAX lattice preserved; prism `SCORING_VERSION` stays 2 — the competition reallocates credits inside the @@ -154,10 +157,12 @@ lands in, not the leaf format or the math).** Per emitted epoch set: credited to the arch's **owner** — owners are rewarded when anyone trains well on their architecture, including in a later epoch than their own submission. -- *emission*: per hotkey `max(own credits, owner credits)` — **max, never +- *per-hotkey credit*: `max(own credits, owner credits)` — **max, never summed**, so the lattice bound and the no-double-count property hold by construction. `Score(0)` rows (cheat/copy-gate) never set an arch's best; hotkeys whose rows are all `NoScore` keep their absence. +- *WTA emission*: argmax over positive per-hotkey credits → one Score leaf; + Prism's emission share (50% of the subnet) goes entirely to that winner. **Top-model publish.** The master tracks the global best bpb across all scored submissions. On a new global best (≤ best ever and < last published), @@ -171,19 +176,24 @@ absent/empty → publishing is a graceful no-op, scoring is unaffected. ## Agentic anti-cheat + AST + metrics gate -Before any pod or LLM spend, the **pre-LLM copy gate** compares the -candidate `architecture.py` against recent submissions from **other miners** -(byte hash + `challenge-ast` fingerprints; same-`miner_hotkey` and -same-`miner_coldkey` prior art excluded): a byte/AST copy of a -**strictly-earlier** submission (`created_at` ordered) is terminal `rejected` -with `Score(0)` — no pod, no LLM. Ties / unknown timestamps fall through to -the LLM path; the published baseline is exempt (miners start from it). After -measure and the cheap `prism-review` arch-only similarity/quality filters, the -shared `challenge-agentic` loop inspects miner sources with read-only tools -(`list_dir`, `read_file`, `ast_summary`, `ast_diff_nearest`, `read_metrics`) -against an **architecture-only** corpus of baseline + other miners' recent -submissions (same hotkey/coldkey exclusion as the gate). Final judge is the -mandatory `submit_verdict` function-call. +Before any pod rent, **pre-pod screens** (no GPU, no private eval assets) run +in order and terminal-reject with `Score(0)` on hit: + +1. **Pre-LLM copy gate** — candidate `architecture.py` vs recent submissions + from **other miners** (byte hash + `challenge-ast`; same hotkey/coldkey + prior art excluded). Byte/AST copy of a **strictly-earlier** submission is + rejected. Ties / unknown timestamps fall through; baseline is exempt. +2. **Static source cheat** (`challenge_agentic::static_source_cheat`) — + hardcoded `METRICS_JSON=` short-circuit; missing + `prism_telemetry.report` / `finish_evaluation` hooks in `training.py`. +3. **Cheap AST similarity** (`prism-review`) — `Copied` / `Suspicious` hard + zero before rent. + +After measure, the LLM quality review and the shared `challenge-agentic` loop +inspect sources + metrics/receipt with read-only tools (`list_dir`, +`read_file`, `ast_summary`, `ast_diff_nearest`, `read_metrics`) against an +**architecture-only** corpus of baseline + other miners' recent submissions. +Final judge is the mandatory `submit_verdict` function-call. | Verdict | Leaf effect | |---------|-------------| diff --git a/docs/evidence/prism-wta-2026-08-08.md b/docs/evidence/prism-wta-2026-08-08.md new file mode 100644 index 000000000..8b3220f3c --- /dev/null +++ b/docs/evidence/prism-wta-2026-08-08.md @@ -0,0 +1,36 @@ +# Prism WTA incident — 2026-08-08 + +## Before (live sealed bundle) + +`GET https://chain.joinbase.ai/v1/weights/latest` at ~2026-08-08T03:42Z: + +- `sealed: true`, `epoch: 24369`, `emission_shares: {design: 0.5, prism: 0.5}` +- Prism positive Score leaves (prod `raw_weight_snapshot`, challenge=`prism`, epoch=`24369`): + +| hotkey (hex → ss58) | score | role | +|---------------------|------:|------| +| `e82eec45…` → `5HK8uU9LzdZqU749xrbXzui74WwhjhcpPmBdnak6X2ziVskL` | 177155 | champion (own best) | +| `14ed40e8…` → `5CY9HzrqC9P4QV9RUbjLwV1g3csCxQm3nKQ7cwrzouJ4mTM2` | 111595 | challenger train | +| `e6cdd0cf…` → `5HHL1a8tj7wEYVG6dri8NKH3zYR9fo9oBA44eadf2Tcb4wUE` | 111595 | arch-owner credit | + +Root cause: `competition_scores` emitted multiple `Score>0` leaves; aggregator +normalized them **proportionally** inside Prism's 50% share (docs previously +said "lattice-proportional — not WTA"). Math: + +- prism fractions: 177155/400345 ≈ 0.4425 → **0.22125** of subnet +- each of the two 111595 → **0.13937** of subnet +- champion also held a design Score(200000) among 5 equal design winners → **+0.1** +- observed champion total **0.32125** = 0.22125 + 0.1 ✓ + +## Fix + +`prism_registry::apply_wta` after competition credits in `build_epoch_leaves`: +only the argmax positive credit keeps a Score leaf (lex-smallest hotkey on +ties). Expected after deploy + next epoch emit + seal: **one** prism hotkey +with ~0.5 of the vector from Prism (plus any design points they also hold). + +## Also shipped + +- Pre-pod screens: copy gate → static `METRICS_JSON`/telemetry hooks → AST + similarity, before `prism-lium` rent. +- `PRISM_MAX_CONCURRENT_EVALS=8` (CLI default, compose default, `env-prod.yml`). diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 467a68865..8e72c4866 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -112,11 +112,13 @@ as a cross-miner copy. LLM quality is coherence-only, not a grader. **Competition:** per epoch you are credited the max of (a) your own best training result and (b) for each arch you own, that arch's best result by *any* trainer — architecture owners are rewarded -for architectures people win with. Scores first land in the leaf set emitted at the -first chain-epoch boundary **after** your run finalizes (a long train that -crosses epochs is normal — outbox assignment is exactly once). Positive -scores then keep participating in later epochs' competition sets until a -better valid score supersedes them. +for architectures people win with. Emission is **winner-take-all**: only the +single highest credit that epoch receives Prism's share (50% of the subnet); +ties break by lexicographically smallest hotkey. Scores first land in the leaf +set emitted at the first chain-epoch boundary **after** your run finalizes (a +long train that crosses epochs is normal — outbox assignment is exactly once). +Positive scores then keep participating in later epochs' competition sets until +a better valid score supersedes them (WTA still collapses to one leaf winner). The global-best model is published to [`BaseIntelligence/prism`](https://github.com/BaseIntelligence/prism) `top-model/`. See [`PRISM.md`](../PRISM.md). diff --git a/docs/runbooks/prism-enable-lium-and-emission.md b/docs/runbooks/prism-enable-lium-and-emission.md index d001bbad9..dff4c6c9f 100644 --- a/docs/runbooks/prism-enable-lium-and-emission.md +++ b/docs/runbooks/prism-enable-lium-and-emission.md @@ -6,7 +6,8 @@ 2. Mount SSH key for pod access (`~/.config/prism-mission/lium_ssh_ed25519`). 3. Pin public eval image digest in config when Real `exec_eval` is fully wired. 4. Run inventory probe → single rent smoke → terminate → `verify_terminated`. -5. Keep `max_concurrent_evals=1` until lease proven. +5. Prod default is `PRISM_MAX_CONCURRENT_EVALS=8` (orchestrator worker count / + semaphore). Dial down only if the Lium lease pool cannot absorb the load. ## Emission ceremony (shared with design) From a5935e48644f282ed3c0b77f57425639e180d1b4 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:53:40 +0000 Subject: [PATCH 2/7] style: cargo fmt for prism WTA / static checks --- crates/challenge-agentic/src/static_checks.rs | 5 +---- crates/prism-registry/src/competition.rs | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/challenge-agentic/src/static_checks.rs b/crates/challenge-agentic/src/static_checks.rs index 6684bc0f8..103541ec1 100644 --- a/crates/challenge-agentic/src/static_checks.rs +++ b/crates/challenge-agentic/src/static_checks.rs @@ -20,10 +20,7 @@ pub struct StaticCheatHit { /// Order: hardcoded `METRICS_JSON=` short-circuit first, then missing Prism /// telemetry hooks in `training.py`. Returns the first hit. #[must_use] -pub fn static_source_cheat( - architecture_py: &str, - training_py: &str, -) -> Option { +pub fn static_source_cheat(architecture_py: &str, training_py: &str) -> Option { for (path, src) in [ ("architecture.py", architecture_py), ("training.py", training_py), diff --git a/crates/prism-registry/src/competition.rs b/crates/prism-registry/src/competition.rs index 61e70ed34..c670078d6 100644 --- a/crates/prism-registry/src/competition.rs +++ b/crates/prism-registry/src/competition.rs @@ -202,10 +202,7 @@ mod tests { #[test] fn wta_keeps_only_the_argmax_score() { - let rows = vec![ - row("aa", None, 177_155), - row("bb", Some("arch_x"), 111_595), - ]; + let rows = vec![row("aa", None, 177_155), row("bb", Some("arch_x"), 111_595)]; let credits = competition_scores(&rows, &owners(&[("arch_x", "cc")])); // Credits: aa=177155, bb=111595, cc=111595 (owner). let wta = apply_wta(credits); From f48fda21645b9e803cc71162e02314a926bb4626 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:55:10 +0000 Subject: [PATCH 3/7] fix(clippy): backtick error_detail in static_checks docs --- crates/challenge-agentic/src/static_checks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/challenge-agentic/src/static_checks.rs b/crates/challenge-agentic/src/static_checks.rs index 103541ec1..9c1251a8a 100644 --- a/crates/challenge-agentic/src/static_checks.rs +++ b/crates/challenge-agentic/src/static_checks.rs @@ -11,7 +11,7 @@ use crate::types::CheatCode; pub struct StaticCheatHit { /// Cheat taxonomy code. pub code: CheatCode, - /// Human-readable reason (safe to surface in error_detail). + /// Human-readable reason (safe to surface in `error_detail`). pub rationale: String, } From c293c9ed80c0f029b7937ccf77edf44b857a1913 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 03:56:59 +0000 Subject: [PATCH 4/7] refactor(prism): extract pre_pod_screens to satisfy line cap --- crates/prism-challenge/src/orchestrator.rs | 60 ++++++++++++---------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index 02d2d2aa4..d595a61ee 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -334,34 +334,10 @@ impl Orchestrator { let id = row.id.clone(); info!(submission_id = %id, miner = %row.miner_hotkey, "prism eval start"); - // Phase 0: pre-pod cheap screens (no GPU, no private eval assets). - // Copy gate → static cheat patterns → AST similarity. Fail-fast with - // Score(0) so a bad submission never rents a Lium pod (~6h waste). - if self.copy_gate_step(&row).await { + // Phase 0: pre-pod cheap screens (no GPU / private eval assets). + let Some(similarity) = self.pre_pod_screens(&id, &row).await else { return Ok(()); - } - if self.static_source_step(&row).await { - return Ok(()); - } - let similarity = match self.similarity_step(&id, &row).await { - Ok(v) => v, - Err(e) => { - if self.maybe_auto_retry(&row, "ast_infra", &e).await { - return Ok(()); - } - self.fail_terminal(&row, "ast_infra", &e).await; - return Ok(()); - } }; - if matches!( - similarity.kind, - prism_review::SimilarityKind::Copied | prism_review::SimilarityKind::Suspicious - ) { - let detail = format!("pre-pod similarity: {:?}", similarity.kind); - self.reject_pre_pod(&row, Some(similarity), None, detail) - .await; - return Ok(()); - } // Phase 1: provision + recipe exec + terminate (always verified). // Lium/infra failures auto-retry (install class); budget exhaustion is @@ -454,6 +430,38 @@ impl Orchestrator { Ok(()) } + /// Pre-pod screens: copy gate → static cheat → AST similarity. + /// Returns `Some(similarity)` when the row may proceed to Lium rent; + /// `None` when already finalized (rejected / failed / retrying). + async fn pre_pod_screens(&self, id: &str, row: &SubmissionState) -> Option { + if self.copy_gate_step(row).await { + return None; + } + if self.static_source_step(row).await { + return None; + } + let similarity = match self.similarity_step(id, row).await { + Ok(v) => v, + Err(e) => { + if self.maybe_auto_retry(row, "ast_infra", &e).await { + return None; + } + self.fail_terminal(row, "ast_infra", &e).await; + return None; + } + }; + if matches!( + similarity.kind, + prism_review::SimilarityKind::Copied | prism_review::SimilarityKind::Suspicious + ) { + let detail = format!("pre-pod similarity: {:?}", similarity.kind); + self.reject_pre_pod(row, Some(similarity), None, detail) + .await; + return None; + } + Some(similarity) + } + /// Pre-LLM copy gate on `architecture.py`. Returns `true` when the row was /// finalized terminal `rejected` (caller must stop processing). /// From aa8f7532e7dec12f17d27cbccd5aa058db45be84 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 04:02:49 +0000 Subject: [PATCH 5/7] fix(loc-cap): move static cheats to challenge-ast; drop submit shim Keep prism-challenge and challenge-agentic under the 1500 LOC gate while preserving pre-pod static screens and GatewayClient via challenge-common. --- crates/challenge-agentic/src/lib.rs | 7 +- crates/challenge-agentic/src/sim.rs | 6 +- crates/challenge-ast/src/lib.rs | 4 + .../src/source_cheats.rs} | 40 +++-- crates/prism-challenge/src/lib.rs | 10 +- crates/prism-challenge/src/orchestrator.rs | 9 +- crates/prism-challenge/src/submit.rs | 146 ------------------ .../prism-challenge/tests/arch_competition.rs | 3 +- .../prism-challenge/tests/cheat_arch_copy.rs | 3 +- crates/prism-challenge/tests/cheat_metrics.rs | 3 +- crates/prism-challenge/tests/copy_gate.rs | 3 +- .../tests/e2e_orchestrate_sim.rs | 3 +- .../prism-challenge/tests/e2e_sim_pipeline.rs | 11 +- 13 files changed, 52 insertions(+), 196 deletions(-) rename crates/{challenge-agentic/src/static_checks.rs => challenge-ast/src/source_cheats.rs} (68%) delete mode 100644 crates/prism-challenge/src/submit.rs diff --git a/crates/challenge-agentic/src/lib.rs b/crates/challenge-agentic/src/lib.rs index 19bf1db6e..0848f07ff 100644 --- a/crates/challenge-agentic/src/lib.rs +++ b/crates/challenge-agentic/src/lib.rs @@ -18,16 +18,17 @@ mod agent; mod llm; mod prompts; mod sim; -mod static_checks; mod tools; mod types; pub use agent::{AgentConfig, OpenRouterAgent}; -pub use challenge_ast::{copy_gate, CopyGateHit, GateCorpusEntry}; +pub use challenge_ast::{ + copy_gate, static_source_cheat, training_has_telemetry_hooks, CopyGateHit, GateCorpusEntry, + SourceCheatHit, SourceCheatKind, +}; pub use llm::{load_api_key_file, DEFAULT_MODEL}; pub use prompts::{AGENTIC_PROMPT_VERSION, DESIGN_DOMAIN_RULES, PRISM_DOMAIN_RULES}; pub use sim::{SimAgent, SIM_CHEAT_BPS, SIM_SUSPICIOUS_BPS}; -pub use static_checks::{static_source_cheat, training_has_telemetry_hooks, StaticCheatHit}; pub use types::{ AgenticBackend, AgenticError, AgenticVerdict, CheatCode, ContainerReviewRequest, CorpusEntry, ReviewRequest, VerdictKind, OPENROUTER_API_BASE, diff --git a/crates/challenge-agentic/src/sim.rs b/crates/challenge-agentic/src/sim.rs index 9255c7d4e..cdce0a57a 100644 --- a/crates/challenge-agentic/src/sim.rs +++ b/crates/challenge-agentic/src/sim.rs @@ -79,10 +79,6 @@ impl AgenticBackend for SimAgent { return Ok(v); } - // Source-only screens are also available via [`crate::static_source_cheat`] - // for the pre-pod orchestrator path; sim keeps the in-review copies so - // metrics-relative checks and corpus AST still share one backend. - if let Some(v) = pages_scrape_cheat_verdict(req)? { return Ok(v); } @@ -191,7 +187,7 @@ fn telemetry_hooks_verdict( return None; } let (path, src) = primaries.iter().find(|(p, _)| p.ends_with("training.py"))?; - if crate::training_has_telemetry_hooks(src) { + if challenge_ast::training_has_telemetry_hooks(src) { return None; } Some(AgenticVerdict { diff --git a/crates/challenge-ast/src/lib.rs b/crates/challenge-ast/src/lib.rs index e50e6ecc2..50b03ad34 100644 --- a/crates/challenge-ast/src/lib.rs +++ b/crates/challenge-ast/src/lib.rs @@ -9,6 +9,7 @@ mod fingerprint; mod gate; mod similarity; +mod source_cheats; mod walk; pub use fingerprint::{fingerprint_source, AstError, Fingerprint}; @@ -19,6 +20,9 @@ pub use gate::{ pub use similarity::{ similarity_bps, structural_diff_summary, summarize_fingerprint, top_k_nearest, Neighbor, }; +pub use source_cheats::{ + static_source_cheat, training_has_telemetry_hooks, SourceCheatHit, SourceCheatKind, +}; /// Crate identity smoke. #[must_use] diff --git a/crates/challenge-agentic/src/static_checks.rs b/crates/challenge-ast/src/source_cheats.rs similarity index 68% rename from crates/challenge-agentic/src/static_checks.rs rename to crates/challenge-ast/src/source_cheats.rs index 9c1251a8a..4c2bd624a 100644 --- a/crates/challenge-agentic/src/static_checks.rs +++ b/crates/challenge-ast/src/source_cheats.rs @@ -1,40 +1,40 @@ //! Cheap source-only cheat screens (no GPU, no private eval assets). -//! -//! Run these **before** renting a Lium pod so a bad submission fails fast -//! instead of burning hours of GPU. Metrics/receipt consistency checks stay -//! post-eval (they need harness output). -use crate::types::CheatCode; +/// Kind of static source hit (maps to agentic `CheatCode` at the call site). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceCheatKind { + /// Hardcoded `METRICS_JSON=` short-circuit. + EvalShortCircuit, + /// Missing Prism telemetry hooks in `training.py`. + MissingTelemetryHooks, +} /// One static source finding. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct StaticCheatHit { - /// Cheat taxonomy code. - pub code: CheatCode, +pub struct SourceCheatHit { + /// Cheat kind. + pub kind: SourceCheatKind, /// Human-readable reason (safe to surface in `error_detail`). pub rationale: String, } /// Scan miner sources for cheap, deterministic cheat patterns. -/// -/// Order: hardcoded `METRICS_JSON=` short-circuit first, then missing Prism -/// telemetry hooks in `training.py`. Returns the first hit. #[must_use] -pub fn static_source_cheat(architecture_py: &str, training_py: &str) -> Option { +pub fn static_source_cheat(architecture_py: &str, training_py: &str) -> Option { for (path, src) in [ ("architecture.py", architecture_py), ("training.py", training_py), ] { if src.contains("METRICS_JSON=") { - return Some(StaticCheatHit { - code: CheatCode::EvalShortCircuit, + return Some(SourceCheatHit { + kind: SourceCheatKind::EvalShortCircuit, rationale: format!("static: hardcoded METRICS_JSON in {path}"), }); } } if !training_has_telemetry_hooks(training_py) { - return Some(StaticCheatHit { - code: CheatCode::MissingTelemetryHooks, + return Some(SourceCheatHit { + kind: SourceCheatKind::MissingTelemetryHooks, rationale: "static: training.py missing prism_telemetry report/finish_evaluation hooks" .into(), }); @@ -48,9 +48,7 @@ pub fn training_has_telemetry_hooks(training_py: &str) -> bool { let imports_shim = training_py.contains("prism_telemetry") || training_py.contains("ctx[\"telemetry\"]") || training_py.contains("ctx['telemetry']"); - let calls_report = training_py.contains(".report("); - let calls_finish = training_py.contains("finish_evaluation("); - imports_shim && calls_report && calls_finish + training_py.contains(".report(") && training_py.contains("finish_evaluation(") && imports_shim } #[cfg(test)] @@ -64,7 +62,7 @@ mod tests { "def train(m, ctx):\n print('METRICS_JSON={}')\n", ) .expect("hit"); - assert_eq!(hit.code, CheatCode::EvalShortCircuit); + assert_eq!(hit.kind, SourceCheatKind::EvalShortCircuit); } #[test] @@ -74,7 +72,7 @@ mod tests { "def train(m, ctx):\n return {}\n", ) .expect("hit"); - assert_eq!(hit.code, CheatCode::MissingTelemetryHooks); + assert_eq!(hit.kind, SourceCheatKind::MissingTelemetryHooks); } #[test] diff --git a/crates/prism-challenge/src/lib.rs b/crates/prism-challenge/src/lib.rs index 60489141c..c365d0991 100644 --- a/crates/prism-challenge/src/lib.rs +++ b/crates/prism-challenge/src/lib.rs @@ -16,10 +16,13 @@ mod api; mod leaf_emit; pub mod orchestrator; mod score; -mod submit; pub use api::{record_epoch, submission_router, AppState}; -pub use leaf_emit::{emit_signed_leaf_set, public_key_from_secret, verify_leaf_sig, LeafEmitError}; +pub use challenge_common::{ + public_key_from_secret, submit_signed_leaf_set, verify_leaf_sig, GatewayClient, + GatewayClientConfig, LeafEmitError, SubmitError, SubmitOutcome, +}; +pub use leaf_emit::emit_signed_leaf_set; pub use orchestrator::{Orchestrator, OrchestratorConfig}; pub use prism_challenge_task::{ CHALLENGE_ID, CHALLENGE_ID_BYTES, SCORE_MAX, SCORING_VERSION, TASK_ID_DOMAIN, @@ -36,9 +39,6 @@ pub use prism_store::{ StoreError, SubmissionState, }; pub use score::{combine_final, FinalOutcome}; -pub use submit::{ - submit_signed_leaf_set, GatewayClient, GatewayClientConfig, SubmitError, SubmitOutcome, -}; pub use bundle::{LeafV1, NoScoreReasonCode, ScoreOrAbsence}; pub use crypto::KEY_LEN; diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index d595a61ee..f332d8564 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -22,7 +22,7 @@ use chain::ChainClient; use challenge_agentic::{ copy_gate, static_source_cheat, AgenticBackend, AgenticVerdict, VerdictKind, }; -use challenge_common::{expected_set_at_chain, PinnedBlockHash}; +use challenge_common::{expected_set_at_chain, GatewayClient, PinnedBlockHash}; use crypto::KEY_LEN; use prism_emit::EpochEmitter; use prism_lium::{EvalJobBackend, InstanceSpec}; @@ -35,7 +35,6 @@ use tracing::{info, warn}; use crate::agentic::{build_review_request, corpus_from_rows, gate_corpus_from_rows, same_miner}; use crate::score::{combine_final, FinalOutcome}; -use crate::submit::GatewayClient; use prism_store::{FinalScore, PrismStore, Stage, StageEvent, StatePatch, SubmissionState}; /// Worker + emitter settings. @@ -115,7 +114,7 @@ impl Orchestrator { chain: Arc, sk: [u8; KEY_LEN], ) -> Self { - let emitter = EpochEmitter::new(Arc::clone(&store), sk, cfg.netuid, gateway.common()); + let emitter = EpochEmitter::new(Arc::clone(&store), sk, cfg.netuid, gateway.clone()); Self { cfg, store, @@ -523,7 +522,7 @@ impl Orchestrator { }; warn!( submission_id = %row.id, - code = ?hit.code, + kind = ?hit.kind, rationale = %hit.rationale, "static source cheat rejected (pod skipped)" ); @@ -532,7 +531,7 @@ impl Orchestrator { None, Some(serde_json::json!({ "gate": "static_source", - "cheat_code": format!("{:?}", hit.code), + "cheat_kind": format!("{:?}", hit.kind), "rationale": hit.rationale, })), hit.rationale.clone(), diff --git a/crates/prism-challenge/src/submit.rs b/crates/prism-challenge/src/submit.rs deleted file mode 100644 index 9b2820dbd..000000000 --- a/crates/prism-challenge/src/submit.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Gateway raw-weights submit client (thin wrap over `challenge-common`). - -use std::collections::BTreeMap; -use std::time::Duration; - -use bundle::LeafV1; -use challenge_common::{ - submit_signed_leaf_set as submit_common, GatewayClient as CommonGatewayClient, - GatewayClientConfig as CommonGatewayClientConfig, SubmitError as CommonSubmitError, - SubmitOutcome as CommonSubmitOutcome, DEFAULT_MAX_RETRIES, DRY_RUN_BASE_URL, -}; -use crypto::KEY_LEN; -use thiserror::Error; - -/// Gateway client config (prism-facing field names). -#[derive(Debug, Clone)] -pub struct GatewayClientConfig { - /// Base URL, or [`DRY_RUN_BASE_URL`] for no network. - pub base_url: String, - /// Max retries after the first attempt. Converted to common `max_attempts = max_retries + 1` - /// to match the prior `0..=max_retries` loop. - pub max_retries: u32, -} - -impl Default for GatewayClientConfig { - fn default() -> Self { - Self { - base_url: "http://127.0.0.1:8080".into(), - max_retries: DEFAULT_MAX_RETRIES, - } - } -} - -impl From for CommonGatewayClientConfig { - fn from(cfg: GatewayClientConfig) -> Self { - CommonGatewayClientConfig { - base_url: cfg.base_url, - max_attempts: cfg.max_retries.saturating_add(1).max(1), - backoff: Duration::from_millis(50), - } - } -} - -/// Submit errors (prism-facing). -#[derive(Debug, Error)] -pub enum SubmitError { - #[error("http: {0}")] - Http(String), - #[error("gateway rejected: {0}")] - Rejected(String), - #[error("serialize: {0}")] - Serialize(String), -} - -impl From for SubmitError { - fn from(err: CommonSubmitError) -> Self { - match err { - CommonSubmitError::Transport(s) => Self::Http(s), - CommonSubmitError::Http { status, body } => { - Self::Rejected(format!("status {status}: {body}")) - } - CommonSubmitError::Serialize(s) => Self::Serialize(s), - } - } -} - -/// Outcome of a submit attempt (prism-facing aggregate). -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum SubmitOutcome { - Accepted, - DryRun { leaf_count: usize }, -} - -/// Thin HTTP client for `POST /v1/weights/raw`. -#[derive(Debug, Clone)] -pub struct GatewayClient { - inner: CommonGatewayClient, -} - -impl GatewayClient { - /// Build client. - /// - /// # Errors - /// HTTP client build failure. - pub fn new(cfg: GatewayClientConfig) -> Result { - let inner = CommonGatewayClient::new(cfg.into())?; - Ok(Self { inner }) - } - - /// Dry-run: no network, reports leaf count. - #[must_use] - pub fn dry_run(leaves: &BTreeMap<[u8; KEY_LEN], LeafV1>) -> SubmitOutcome { - match CommonGatewayClient::dry_run(leaves) { - CommonSubmitOutcome::DryRun { leaf_count } => SubmitOutcome::DryRun { leaf_count }, - _ => SubmitOutcome::DryRun { - leaf_count: leaves.len(), - }, - } - } - - /// Whether this client skips network I/O. - #[must_use] - pub fn is_dry_run(&self) -> bool { - self.inner.is_dry_run() - } - - /// Shared-client clone for the epoch emitter (`prism-emit`). - #[must_use] - pub fn common(&self) -> CommonGatewayClient { - self.inner.clone() - } -} - -/// Submit signed leaves (or dry-run when `base_url` is `dry-run`). -/// -/// `challenge_id` / `epoch` are accepted for call-site compatibility; leaves already -/// carry both. -/// -/// # Errors -/// HTTP / rejection. -pub async fn submit_signed_leaf_set( - client: &GatewayClient, - _challenge_id: &str, - _epoch: u64, - leaves: &BTreeMap<[u8; KEY_LEN], LeafV1>, -) -> Result { - if client.is_dry_run() || client.inner.base_url() == DRY_RUN_BASE_URL { - return Ok(GatewayClient::dry_run(leaves)); - } - let _outcomes = submit_common(&client.inner, leaves).await?; - Ok(SubmitOutcome::Accepted) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn dry_run_counts() { - let m = BTreeMap::new(); - assert_eq!( - GatewayClient::dry_run(&m), - SubmitOutcome::DryRun { leaf_count: 0 } - ); - } -} diff --git a/crates/prism-challenge/tests/arch_competition.rs b/crates/prism-challenge/tests/arch_competition.rs index 8d7183d84..74217dfa7 100644 --- a/crates/prism-challenge/tests/arch_competition.rs +++ b/crates/prism-challenge/tests/arch_competition.rs @@ -138,7 +138,8 @@ fn mk_orchestrator( let gateway = Arc::new( GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .unwrap(), ); diff --git a/crates/prism-challenge/tests/cheat_arch_copy.rs b/crates/prism-challenge/tests/cheat_arch_copy.rs index 0b25185c3..b6e7f9688 100644 --- a/crates/prism-challenge/tests/cheat_arch_copy.rs +++ b/crates/prism-challenge/tests/cheat_arch_copy.rs @@ -83,7 +83,8 @@ async fn baseline_arch_train_copy_scores_zero() { let gateway = Arc::new( GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .unwrap(), ); diff --git a/crates/prism-challenge/tests/cheat_metrics.rs b/crates/prism-challenge/tests/cheat_metrics.rs index 97deaa6c9..13d74e300 100644 --- a/crates/prism-challenge/tests/cheat_metrics.rs +++ b/crates/prism-challenge/tests/cheat_metrics.rs @@ -82,7 +82,8 @@ async fn hardcoded_metrics_json_scores_zero() { let gateway = Arc::new( GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .unwrap(), ); diff --git a/crates/prism-challenge/tests/copy_gate.rs b/crates/prism-challenge/tests/copy_gate.rs index 103cb61a8..353493fd1 100644 --- a/crates/prism-challenge/tests/copy_gate.rs +++ b/crates/prism-challenge/tests/copy_gate.rs @@ -127,7 +127,8 @@ fn mk_orchestrator( let gateway = Arc::new( GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .unwrap(), ); diff --git a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs index efa7dd6bc..ff8b39028 100644 --- a/crates/prism-challenge/tests/e2e_orchestrate_sim.rs +++ b/crates/prism-challenge/tests/e2e_orchestrate_sim.rs @@ -94,7 +94,8 @@ fn mk_orchestrator( let gateway = Arc::new( GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .unwrap(), ); diff --git a/crates/prism-challenge/tests/e2e_sim_pipeline.rs b/crates/prism-challenge/tests/e2e_sim_pipeline.rs index 47af30e5e..2ca1384b5 100644 --- a/crates/prism-challenge/tests/e2e_sim_pipeline.rs +++ b/crates/prism-challenge/tests/e2e_sim_pipeline.rs @@ -64,15 +64,14 @@ async fn e2e_sim_happy_path_scores_and_emits_d24() { let gw = GatewayClient::new(GatewayClientConfig { base_url: "dry-run".into(), - max_retries: 0, + max_attempts: 1, + backoff: std::time::Duration::from_millis(1), }) .expect("gw"); - let out = submit_signed_leaf_set(&gw, CHALLENGE_ID, 7, &leaves) - .await - .expect("submit"); + let out = submit_signed_leaf_set(&gw, &leaves).await.expect("submit"); assert!(matches!( - out, - prism_challenge::SubmitOutcome::DryRun { leaf_count: 1 } + out.as_slice(), + [prism_challenge::SubmitOutcome::DryRun { leaf_count: 1 }] )); } From d1948d2c05e43c23215d16cd61f3319278b9ab44 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 04:04:25 +0000 Subject: [PATCH 6/7] fix(clippy): drop unused re-exports in prism leaf_emit --- crates/prism-challenge/src/leaf_emit.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/prism-challenge/src/leaf_emit.rs b/crates/prism-challenge/src/leaf_emit.rs index 69071f9ad..c45b4db37 100644 --- a/crates/prism-challenge/src/leaf_emit.rs +++ b/crates/prism-challenge/src/leaf_emit.rs @@ -7,7 +7,7 @@ use challenge_common::{emit_signed_leaf_set as emit_signed_leaf_set_common, Hotk use crypto::KEY_LEN; use prism_challenge_task::CHALLENGE_ID_BYTES; -pub use challenge_common::{public_key_from_secret, verify_leaf_sig, LeafEmitError}; +use challenge_common::LeafEmitError; /// Sign exactly one leaf per `h ∈ expected` under `prism`. Refuses subset/superset (D24). /// @@ -27,6 +27,7 @@ mod tests { #![allow(clippy::unwrap_used)] use super::*; use bundle::ScoreOrAbsence; + use challenge_common::{public_key_from_secret, verify_leaf_sig}; use crypto::KEY_LEN; fn sk() -> [u8; KEY_LEN] { From 63288fc4c0355a2c3fbebb178e4d958bef83583d Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 04:05:58 +0000 Subject: [PATCH 7/7] fix(clippy): drop unused CHALLENGE_ID import in e2e_sim_pipeline --- crates/prism-challenge/tests/e2e_sim_pipeline.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/prism-challenge/tests/e2e_sim_pipeline.rs b/crates/prism-challenge/tests/e2e_sim_pipeline.rs index 2ca1384b5..c467825d6 100644 --- a/crates/prism-challenge/tests/e2e_sim_pipeline.rs +++ b/crates/prism-challenge/tests/e2e_sim_pipeline.rs @@ -10,7 +10,7 @@ use crypto::KEY_LEN; use prism_challenge::{ emit_signed_leaf_set, example_valid_request, run_sim_pipeline, score_from_pipeline, submit_signed_leaf_set, GatewayClient, GatewayClientConfig, PipelineInput, PipelineOutcome, - PrismConfig, SubmissionService, CHALLENGE_ID, SCORE_MAX, + PrismConfig, SubmissionService, SCORE_MAX, }; use prism_lium::{EvalJobBackend, SimLiumBackend};