From 8b37134763c8d5373c9df969e1a6acb26e73996f Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 08:15:24 +0000 Subject: [PATCH] fix(prism): fair review for generic LM tropes Stop Score(0) on RMSNorm/RoPE/SwiGLU false positives; champions-only copy corpus; public gallery shows tops/ex-tops only. --- crates/challenge-agentic/src/prompts.rs | 14 +- crates/db/src/prism_store.rs | 20 +++ crates/prism-challenge/src/api.rs | 2 +- crates/prism-challenge/src/orchestrator.rs | 22 +-- crates/prism-challenge/src/score.rs | 25 +++- crates/prism-challenge/tests/copy_gate.rs | 5 +- crates/prism-pipeline/src/precheck.rs | 5 +- crates/prism-review/prompts/similarity_v3.md | 56 ++++++++ crates/prism-review/src/generic_arch.rs | 136 +++++++++++++++++++ crates/prism-review/src/lib.rs | 2 + crates/prism-review/src/llm.rs | 18 ++- crates/prism-review/src/prompts.rs | 8 +- crates/prism-store/src/dbprism.rs | 7 + crates/prism-store/src/store.rs | 36 +++++ crates/site-api/src/handlers.rs | 12 +- crates/site-data/src/map.rs | 39 ++++-- docs/COMPLETENESS.md | 2 +- docs/PRISM.md | 72 +++++----- docs/PRISM_RECIPE.md | 27 ++-- docs/SITE_API.md | 5 +- docs/external-miner/prism.md | 13 +- 21 files changed, 433 insertions(+), 93 deletions(-) create mode 100644 crates/prism-review/prompts/similarity_v3.md create mode 100644 crates/prism-review/src/generic_arch.rs diff --git a/crates/challenge-agentic/src/prompts.rs b/crates/challenge-agentic/src/prompts.rs index b34bf9b04..c2603ed7c 100644 --- a/crates/challenge-agentic/src/prompts.rs +++ b/crates/challenge-agentic/src/prompts.rs @@ -1,16 +1,18 @@ //! System prompt for the agentic anti-cheat loop. /// Prompt version stamp (audit). -pub const AGENTIC_PROMPT_VERSION: &str = "agentic_v1"; +pub const AGENTIC_PROMPT_VERSION: &str = "agentic_v2"; /// Prism-specific anti-cheat rules (metrics / eval short-circuit / AST copy / /// telemetry-hook contract). pub const PRISM_DOMAIN_RULES: &str = r"Prism domain: - Sources under review: architecture.py + training.py (miner code only). +- Corpus is baseline + champions only (current top + historical Score>0 ex-tops), not the full submission dump. - Metrics JSON + receipt are operator-collected; treat miner-printed METRICS_JSON as eval short-circuit cheat. - Telemetry contract (recipe >= 1.1.0): training.py MUST import prism_telemetry and call prism_telemetry.report(loss=..., step=..., ...) during training plus prism_telemetry.finish_evaluation() to end the eval. Missing hooks = contract violation -> cheat (missing_telemetry_hooks). -- Cheat if: near-identical AST/byte copy of a corpus architecture.py (training.py is exempt from similarity — the same training script on a different architecture is fine); hardcoded METRICS_JSON in miner sources; bpb impossible vs tokens_seen/wall_clock (e.g. bpb<<1 with ~0 tokens, or tokens_seen=0 with a finite bpb). -- suspicious: strong structural overlap or inconsistent metrics without a slam-dunk forge. +- Cheat if: near-identical AST/byte copy of a corpus architecture.py (AST similarity_bps >= 9500; training.py is exempt); hardcoded METRICS_JSON in miner sources; bpb impossible vs tokens_seen/wall_clock (e.g. bpb<<1 with ~0 tokens, or tokens_seen=0 with a finite bpb); eval peeking / telemetry bypass. +- NEVER treat standard modern LM components as plagiarism: RMSNorm, LayerNorm, RoPE/ALiBi, SwiGLU/GeGLU, gated residual, parallel residual, GQA/MQA, Pre-Norm/Post-Norm. Those are public recipe ingredients, not cheats. +- suspicious: only for strong unique structural overlap with a champion (AST >= 8500) or inconsistent metrics without a slam-dunk forge. Below AST 8500 with no other cheat signal → clean. - Quality/coherence of the model is NOT your job — only anti-cheat."; /// Design-specific anti-cheat rules (harness / pages / sanitize). @@ -46,8 +48,10 @@ Workflow: Verdict policy: - clean: original enough; empty cheat_codes -- suspicious: strong overlap / odd metrics — still Score(0) -- cheat: clear copy / forgery — Score(0); set cheat_codes + nearest_id + similarity_bps +- suspicious: unique structural overlap with a corpus entry (AST band) or odd metrics — still Score(0) +- cheat: clear copy / forgery / eval short-circuit — Score(0); set cheat_codes + nearest_id + similarity_bps + +Do NOT put RMSNorm / RoPE / SwiGLU / LayerNorm / gated or parallel residual in rationale as copy evidence. AST similarity_bps thresholds (must match SimAgent; use ast_diff_nearest): - >= 9500 → cheat (ast_architecture_copy or near_identical_harness_copy) diff --git a/crates/db/src/prism_store.rs b/crates/db/src/prism_store.rs index b2fc6cead..32fb39886 100644 --- a/crates/db/src/prism_store.rs +++ b/crates/db/src/prism_store.rs @@ -286,6 +286,26 @@ pub async fn list_prism_submissions( Ok(rows) } +/// Champion corpus: historical Score>0 WTA/leaf winners (current top + ex-tops). +/// +/// # Errors +/// SQL error. +pub async fn list_prism_champions( + pool: &PgPool, + limit: i64, +) -> Result, DbError> { + let q = format!( + "SELECT {COLS} FROM prism_submission \ + WHERE kind = 'score' AND score > 0 \ + ORDER BY created_at DESC LIMIT $1" + ); + let rows = sqlx::query_as::<_, PrismSubmissionRow>(&q) + .bind(limit) + .fetch_all(pool) + .await?; + Ok(rows) +} + /// Ascending event journal for one row. /// /// # Errors diff --git a/crates/prism-challenge/src/api.rs b/crates/prism-challenge/src/api.rs index 4f3f88704..2c9f4d463 100644 --- a/crates/prism-challenge/src/api.rs +++ b/crates/prism-challenge/src/api.rs @@ -284,7 +284,7 @@ async fn post_precheck( &req.architecture_py, now_ms(), ); - let recent = st.store.list(None, None, 64).await.unwrap_or_default(); + let recent = st.store.list_champions(64).await.unwrap_or_default(); let result = evaluate_copy_precheck(&candidate, &recent, quota); Json(precheck_json(&result)).into_response() } diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index ec1161361..7466e2d42 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -56,7 +56,7 @@ pub struct OrchestratorConfig { pub emit_poll: Duration, /// Rent attempt budget before `failed`. pub max_attempts: u32, - /// Similarity corpus size (recent submissions + baseline). + /// Similarity / agentic corpus size (champions + baseline). pub similarity_corpus_limit: u32, /// Stuck sweep grace (seconds). Must exceed max healthy wall-clock of a /// live worker hold: `PRISM_SSH_RUNNING_TIMEOUT` (≤15m) + train cap (6h) + @@ -457,10 +457,10 @@ impl Orchestrator { return None; } }; - if matches!( - similarity.kind, - prism_review::SimilarityKind::Copied | prism_review::SimilarityKind::Suspicious - ) { + // Only hard-reject LLM `Copied`. `Suspicious` is advisory — agentic + // (post-pod, AST-thresholded) is the primary judge. Generic LM tropes + // are coerced to Original in prism-review parsers. + if matches!(similarity.kind, prism_review::SimilarityKind::Copied) { let detail = format!("pre-pod similarity: {:?}", similarity.kind); self.reject_pre_pod(row, Some(similarity), None, detail) .await; @@ -472,7 +472,7 @@ impl Orchestrator { /// Pre-LLM copy gate on `architecture.py`. Returns `true` when the row was /// finalized terminal `rejected` (caller must stop processing). /// - /// The corpus is recent submissions (any status — prior art is prior art) + /// The corpus is **champions** (Score>0 current top + historical ex-tops), /// ordered by store `created_at`; the published baseline is exempt by id /// prefix inside [`copy_gate`]. Ties / unknown timestamps fall through to /// the LLM similarity review. Training-only rows (`arch_id` set) skip the @@ -481,7 +481,11 @@ impl Orchestrator { if row.arch_id.is_some() { return false; } - let recent = self.store.list(None, None, 64).await.unwrap_or_default(); + let recent = self + .store + .list_champions(self.cfg.similarity_corpus_limit.max(64)) + .await + .unwrap_or_default(); let corpus = gate_corpus_from_rows(row, &recent); let Some(hit) = copy_gate(&row.architecture_py, row.created_at_ms, &corpus) else { return false; @@ -744,7 +748,7 @@ impl Orchestrator { }; let recent = self .store - .list(Some("terminated"), None, self.cfg.similarity_corpus_limit) + .list_champions(self.cfg.similarity_corpus_limit) .await .unwrap_or_default(); // Training-only rows: drop the referenced registry arch from the @@ -811,7 +815,7 @@ impl Orchestrator { async fn similarity_corpus(&self, candidate: &SubmissionState) -> Vec { let recent = self .store - .list(Some("terminated"), None, self.cfg.similarity_corpus_limit) + .list_champions(self.cfg.similarity_corpus_limit) .await .unwrap_or_default(); let mut v = vec![SourceSnippet { diff --git a/crates/prism-challenge/src/score.rs b/crates/prism-challenge/src/score.rs index e062f3c03..ac2811204 100644 --- a/crates/prism-challenge/src/score.rs +++ b/crates/prism-challenge/src/score.rs @@ -30,8 +30,9 @@ pub enum FinalOutcome { /// The score is **pure bpb**: the LLM review is an anti-cheat / coherence /// GATE, never a grader — its quality vote and issues are recorded as audit /// events but never add nor remove points. Agentic `Cheat`/`Suspicious` and -/// cheap similarity `Copied`/`Suspicious` are hard gates (miner-attributable -/// `Score{0}`). Missing agentic verdict is fail-closed upstream as +/// cheap similarity `Copied` are hard gates (miner-attributable `Score{0}`). +/// Cheap LLM `Suspicious` is advisory only (agentic + AST thresholds are the +/// primary judge). Missing agentic verdict is fail-closed upstream as /// [`FinalOutcome::ChallengeInternal`]. /// /// # Panics @@ -53,10 +54,7 @@ pub fn combine_final(outcome: &FinalOutcome) -> prism_store::FinalScore { if matches!(agentic, VerdictKind::Cheat | VerdictKind::Suspicious) { return FinalScore::Score(0); } - if matches!( - similarity, - prism_review::SimilarityKind::Copied | prism_review::SimilarityKind::Suspicious - ) { + if matches!(similarity, prism_review::SimilarityKind::Copied) { return FinalScore::Score(0); } FinalScore::Score(score_from_bpb(*bpb)) @@ -70,6 +68,7 @@ mod final_tests { use prism_challenge_task::SCORE_MAX; use prism_review::SimilarityKind::Copied; use prism_review::SimilarityKind::Original; + use prism_review::SimilarityKind::Suspicious; #[test] fn copied_is_hard_zero() { @@ -82,6 +81,20 @@ mod final_tests { assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0)); } + #[test] + fn cheap_suspicious_is_not_hard_zero() { + let o = FinalOutcome::Measured { + bpb: 1.0, + quality: 900, + similarity: Suspicious, + agentic: challenge_agentic::VerdictKind::Clean, + }; + assert!(matches!( + combine_final(&o), + prism_store::FinalScore::Score(v) if v > 0 + )); + } + #[test] fn agentic_cheat_is_hard_zero() { let o = FinalOutcome::Measured { diff --git a/crates/prism-challenge/tests/copy_gate.rs b/crates/prism-challenge/tests/copy_gate.rs index 353493fd1..6b8f1cc52 100644 --- a/crates/prism-challenge/tests/copy_gate.rs +++ b/crates/prism-challenge/tests/copy_gate.rs @@ -94,6 +94,9 @@ fn row( status: Stage, created_ms: u64, ) -> SubmissionState { + // Terminated priors used as copy-gate victims must be champions (Score>0); + // the gate corpus is tops + ex-tops only. + let final_score = matches!(status, Stage::Terminated).then_some(FinalScore::Score(1)); SubmissionState { id: id.into(), miner_hotkey: hotkey.into(), @@ -112,7 +115,7 @@ fn row( arch_id: None, review: None, similarity: None, - final_score: None, + final_score, retry_count: 0, error_detail: None, created_at_ms: created_ms, diff --git a/crates/prism-pipeline/src/precheck.rs b/crates/prism-pipeline/src/precheck.rs index 948f4ca90..e0c01cf38 100644 --- a/crates/prism-pipeline/src/precheck.rs +++ b/crates/prism-pipeline/src/precheck.rs @@ -115,7 +115,8 @@ pub fn same_miner(candidate: &SubmissionState, other: &SubmissionState) -> bool ) } -/// Pre-LLM copy-gate corpus: other miners' prior art only (hotkey + coldkey). +/// Pre-LLM copy-gate corpus: other miners' **champion** prior art only +/// (caller should pass `PrismStore::list_champions`; hotkey + coldkey excluded). #[must_use] pub fn gate_corpus_from_rows( candidate: &SubmissionState, @@ -132,7 +133,7 @@ pub fn gate_corpus_from_rows( .collect() } -/// Baseline + recent terminated submissions as agentic corpus entries. +/// Baseline + champion submissions as agentic corpus entries. /// /// Architecture.py only; same-hotkey and same-coldkey prior art excluded. #[must_use] diff --git a/crates/prism-review/prompts/similarity_v3.md b/crates/prism-review/prompts/similarity_v3.md new file mode 100644 index 000000000..e37a14412 --- /dev/null +++ b/crates/prism-review/prompts/similarity_v3.md @@ -0,0 +1,56 @@ +You are the PRISM architecture similarity judge (anti-farming) for a +pretraining-recipe challenge. + +The CANDIDATE below is the `architecture.py` of a miner submission. The +CORPUS lists reference architectures: the operator `baseline` and **champion** +architectures only (current top + historical ex-tops with Score>0). Decide +whether the CANDIDATE architecture is effectively a copy of any corpus +architecture. + +SCOPE: judge `architecture.py` ONLY. The companion `training.py` is NOT part +of this judgment — the same training script on two different architectures +is legitimate, and the same architecture with a different training script is +still an architecture copy. + +Definitions: +- `copied`: near-verbatim architecture, or trivial renaming/formatting + shuffles, or the same model definition with cosmetic deltas (renamed + identifiers, reordered methods, comment edits). Hard zero. +- `suspicious`: strong structural overlap of a *specific* corpus model + (same unique layer stack, same shapes, same forward flow, same custom + blocks) but rewritten enough to blur. Use sparingly — this is advisory + only for operators; do NOT use it for shared modern-LM vocabulary. +- `original`: normal engineering resemblance, standard components, or clear + novelty. + +HARD BAN — these are standard modern LM components and MUST NEVER appear as +copy/suspicious evidence by themselves (alone or together): +RMSNorm, LayerNorm, BatchNorm, GroupNorm, Rotary / RoPE / ALiBi / absolute / +relative positional embeddings, SwiGLU / GeGLU / GLU / GELU / SiLU feed-forward, +multi-head / GQA / MQA attention, KV cache, gated residual, parallel residual / +parallel MLP+attention blocks, Pre-Norm / Post-Norm, weight tying, dropout, +FlashAttention, MoE routers, depthwise / causal convolutions used as PE. +Citing any of the above as evidence of copying is a judge error → output +`original` instead. + +Only flag `copied` / `suspicious` when the candidate mirrors a *particular* +corpus entry's unique structure (same custom block composition, same unusual +tensor shapes / depths / widths, same novel wiring), not when both use the +same public recipe ingredients. + +Shuffling the order of functions/classes does NOT matter. + +Output STRICT JSON only: +{"kind": "original|suspicious|copied", + "score": float 0..1, + "closest": "", + "evidence": [str, str, str]} +evidence: at most 3 short strings, no markdown. Evidence must name +candidate-specific overlap with a corpus id — never a generic component name +from the ban list above. + +=== CANDIDATE architecture.py === +{ARCH} + +=== CORPUS (architecture.py only; champions + baseline) === +{CORPUS} diff --git a/crates/prism-review/src/generic_arch.rs b/crates/prism-review/src/generic_arch.rs new file mode 100644 index 000000000..80a53a2d5 --- /dev/null +++ b/crates/prism-review/src/generic_arch.rs @@ -0,0 +1,136 @@ +//! Post-parse guard: LLM similarity must not Score(0) on generic LM tropes. + +use crate::types::{SimilarityKind, SimilarityVerdict}; + +/// Substrings that are standard modern-LM vocabulary (case-insensitive). +/// When *every* evidence line matches one of these (and nothing else +/// substantive), a `suspicious` / soft-`copied` verdict is coerced to +/// `original`. +const GENERIC_TROPE_NEEDLES: &[&str] = &[ + "rmsnorm", + "layer norm", + "layernorm", + "batchnorm", + "groupnorm", + "rotary", + "rope", + "alibi", + "positional embed", + "swiglu", + "geglu", + "gated residual", + "parallel residual", + "feed-forward", + "feed forward", + "multi-head attention", + "multihead attention", + "grouped-query", + "gqa", + "mqa", + "pre-norm", + "post-norm", + "weight ty", + "flashattention", + "flash attention", +]; + +/// True when `evidence` is non-empty and every line is only a generic trope. +#[must_use] +pub fn evidence_is_only_generic_tropes(evidence: &[String]) -> bool { + if evidence.is_empty() { + return false; + } + evidence.iter().all(|e| { + let lower = e.to_ascii_lowercase(); + let trimmed = lower.trim(); + if trimmed.is_empty() { + return true; + } + GENERIC_TROPE_NEEDLES.iter().any(|n| trimmed.contains(n)) + }) +} + +/// Coerce false-positive LLM similarity verdicts that cite only standard +/// components. Hard `copied` with score ≥ 0.95 is left alone (near-verbatim +/// copies can still mention shared blocks in evidence). +#[must_use] +pub fn coerce_generic_similarity(mut v: SimilarityVerdict) -> SimilarityVerdict { + let only_generic = evidence_is_only_generic_tropes(&v.evidence); + match v.kind { + SimilarityKind::Suspicious if only_generic || v.score < 0.9 => { + v.kind = SimilarityKind::Original; + if only_generic { + v.evidence.insert( + 0, + "coerced: generic LM components are not plagiarism".into(), + ); + v.evidence.truncate(3); + } + } + SimilarityKind::Copied if only_generic && v.score < 0.95 => { + v.kind = SimilarityKind::Original; + v.evidence.insert( + 0, + "coerced: generic LM components are not plagiarism".into(), + ); + v.evidence.truncate(3); + } + _ => {} + } + v +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use crate::prompts::SIMILARITY_PROMPT_VERSION; + + fn verdict(kind: SimilarityKind, score: f64, evidence: &[&str]) -> SimilarityVerdict { + SimilarityVerdict { + kind, + score, + closest: Some("subm:deadbeef".into()), + evidence: evidence.iter().map(|s| (*s).to_owned()).collect(), + prompt_version: SIMILARITY_PROMPT_VERSION, + } + } + + #[test] + fn tropes_coerce_suspicious() { + let v = coerce_generic_similarity(verdict( + SimilarityKind::Suspicious, + 0.7, + &[ + "RMSNorm usage", + "Rotary embeddings", + "Gated residual connections", + ], + )); + assert!(matches!(v.kind, SimilarityKind::Original)); + } + + #[test] + fn tropes_coerce_swiglu_case() { + let v = coerce_generic_similarity(verdict( + SimilarityKind::Suspicious, + 0.7, + &[ + "Parallel residual blocks", + "SwiGLU feed-forward", + "Layer normalization", + ], + )); + assert!(matches!(v.kind, SimilarityKind::Original)); + } + + #[test] + fn unique_structure_keeps_copied() { + let v = coerce_generic_similarity(verdict( + SimilarityKind::Copied, + 0.97, + &["same custom DualPathBlock wiring as subm:aabbccdd"], + )); + assert!(matches!(v.kind, SimilarityKind::Copied)); + } +} diff --git a/crates/prism-review/src/lib.rs b/crates/prism-review/src/lib.rs index c98ade091..03605348c 100644 --- a/crates/prism-review/src/lib.rs +++ b/crates/prism-review/src/lib.rs @@ -16,11 +16,13 @@ #![forbid(unsafe_code)] +mod generic_arch; mod llm; mod prompts; mod sim; mod types; +pub use generic_arch::{coerce_generic_similarity, evidence_is_only_generic_tropes}; pub use llm::{load_api_key_file, OpenRouterClient}; pub use prompts::{REVIEW_PROMPT_VERSION, SIMILARITY_PROMPT_VERSION}; pub use sim::SimReviewer; diff --git a/crates/prism-review/src/llm.rs b/crates/prism-review/src/llm.rs index 88f33c643..684472ffb 100644 --- a/crates/prism-review/src/llm.rs +++ b/crates/prism-review/src/llm.rs @@ -5,8 +5,9 @@ use std::path::PathBuf; use async_trait::async_trait; use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT}; +use crate::generic_arch::coerce_generic_similarity; use crate::prompts::{ - REVIEW_PROMPT_V3, REVIEW_PROMPT_VERSION, SIMILARITY_PROMPT_V2, SIMILARITY_PROMPT_VERSION, + REVIEW_PROMPT_V3, REVIEW_PROMPT_VERSION, SIMILARITY_PROMPT_V3, SIMILARITY_PROMPT_VERSION, }; use crate::types::{ truncate_source, ReviewError, ReviewVerdict, SimilarityKind, SimilarityVerdict, SourceSnippet, @@ -244,13 +245,13 @@ fn parse_similarity(text: &str) -> Result { .collect::>() }) .unwrap_or_default(); - Ok(SimilarityVerdict { + Ok(coerce_generic_similarity(SimilarityVerdict { kind, score, closest, evidence, prompt_version: SIMILARITY_PROMPT_VERSION, - }) + })) } /// Corpus rendering for the similarity prompt: **architectures only** @@ -289,7 +290,7 @@ impl ReviewBackend for OpenRouterClient { architecture_py: &str, corpus: &[SourceSnippet], ) -> Result { - let prompt = SIMILARITY_PROMPT_V2 + let prompt = SIMILARITY_PROMPT_V3 .replace("{ARCH}", &truncate_source(architecture_py)) .replace("{CORPUS}", &corpus_block(corpus)); let answer = self.chat(&prompt).await?; @@ -333,6 +334,15 @@ mod tests { assert_eq!(v.closest.as_deref(), Some("baseline")); } + #[test] + fn parse_similarity_coerces_generic_suspicious() { + let v = parse_similarity( + r#"{"kind":"suspicious","score":0.7,"closest":"subm:89e6273b","evidence":["RMSNorm usage","Rotary embeddings","Gated residual connections"]}"#, + ) + .unwrap(); + assert!(matches!(v.kind, SimilarityKind::Original)); + } + #[test] fn parse_similarity_rejects_garbage() { assert!(parse_similarity(r#"{"kind": "unknown"}"#).is_err()); diff --git a/crates/prism-review/src/prompts.rs b/crates/prism-review/src/prompts.rs index 0a8e4e5a8..6d7a0f9ba 100644 --- a/crates/prism-review/src/prompts.rs +++ b/crates/prism-review/src/prompts.rs @@ -13,9 +13,15 @@ pub const REVIEW_PROMPT_V3: &str = include_str!("../prompts/review_v3.md"); /// v2 scope change: similarity judges `architecture.py` ONLY — `training.py` /// is exempt from both the candidate and the corpus (the same training /// script on two different architectures is legitimate). +/// +/// v3: corpus is champions (top + ex-tops) + baseline; hard ban on citing +/// standard LM components (RMSNorm / RoPE / SwiGLU / …) as plagiarism evidence. +#[allow(dead_code)] // retained for audit / diff against similarity-v3 pub const SIMILARITY_PROMPT_V2: &str = include_str!("../prompts/similarity_v2.md"); +/// Current similarity prompt (v3). +pub const SIMILARITY_PROMPT_V3: &str = include_str!("../prompts/similarity_v3.md"); /// Version string for the review prompt. pub const REVIEW_PROMPT_VERSION: &str = "review-v3"; /// Version string for the similarity prompt. -pub const SIMILARITY_PROMPT_VERSION: &str = "similarity-v2"; +pub const SIMILARITY_PROMPT_VERSION: &str = "similarity-v3"; diff --git a/crates/prism-store/src/dbprism.rs b/crates/prism-store/src/dbprism.rs index 597ff7479..8a6cd7ab5 100644 --- a/crates/prism-store/src/dbprism.rs +++ b/crates/prism-store/src/dbprism.rs @@ -317,6 +317,13 @@ impl PrismStore for DbPrismStore { states_filled(&self.pool, rows).await } + async fn list_champions(&self, limit: u32) -> Result, StoreError> { + let rows = dbs::list_prism_champions(&self.pool, i64::from(limit)) + .await + .map_err(|e| StoreError::Backend(e.to_string()))?; + states_filled(&self.pool, rows).await + } + async fn events(&self, id: &str) -> Result, StoreError> { dbs::prism_stage_events(&self.pool, id) .await diff --git a/crates/prism-store/src/store.rs b/crates/prism-store/src/store.rs index 6cb9fbfb9..075f42cd5 100644 --- a/crates/prism-store/src/store.rs +++ b/crates/prism-store/src/store.rs @@ -292,6 +292,11 @@ pub trait PrismStore: Send + Sync + std::fmt::Debug { limit: u32, ) -> Result, StoreError>; + /// Champion corpus for copy/similarity/agentic gates: submissions with + /// `Score(v)` where `v > 0` (current top + historical WTA ex-tops). + /// Newest first. Does **not** include baseline (callers add that). + async fn list_champions(&self, limit: u32) -> Result, StoreError>; + /// Ascending journal. async fn events(&self, id: &str) -> Result, StoreError>; @@ -584,6 +589,20 @@ impl PrismStore for MemoryPrismStore { Ok(v) } + async fn list_champions(&self, limit: u32) -> Result, StoreError> { + let mut v: Vec<_> = self + .rows + .lock() + .map_err(|_| StoreError::Backend("poison".into()))? + .iter() + .filter(|r| matches!(r.final_score, Some(FinalScore::Score(s)) if s > 0)) + .cloned() + .collect(); + v.sort_by_key(|r| std::cmp::Reverse(r.created_at_ms)); + v.truncate(limit as usize); + Ok(v) + } + async fn events(&self, id: &str) -> Result, StoreError> { Ok(self .events @@ -916,6 +935,23 @@ mod tests { assert_eq!(only_a[0].id, "a"); } + #[tokio::test] + async fn list_champions_score_positive_only() { + let s = MemoryPrismStore::new(); + let mut winner = row("w", "11"); + winner.status = Stage::Terminated; + winner.final_score = Some(FinalScore::Score(42)); + let mut zero = row("z", "22"); + zero.status = Stage::Terminated; + zero.final_score = Some(FinalScore::Score(0)); + s.insert_queued(&winner).await.unwrap(); + s.insert_queued(&zero).await.unwrap(); + s.insert_queued(&row("q", "33")).await.unwrap(); + let champs = s.list_champions(10).await.unwrap(); + assert_eq!(champs.len(), 1); + assert_eq!(champs[0].id, "w"); + } + #[tokio::test] async fn apply_patches_and_journals() { let s = MemoryPrismStore::new(); diff --git a/crates/site-api/src/handlers.rs b/crates/site-api/src/handlers.rs index 63639e062..6c81855ed 100644 --- a/crates/site-api/src/handlers.rs +++ b/crates/site-api/src/handlers.rs @@ -14,8 +14,9 @@ use crate::state::SiteState; use crate::upstream::{self, DESIGN, PRISM}; use site_data::map::{ activity_from_lives, design_arena_from_dashboard, design_leaderboard, design_submission, - leaderboard_matches_query, list_arenas, prism_arena_from_live, prism_bpb_leaderboard, - prism_submission, prism_telemetry, prism_window, submission_matches_query, + is_prism_champion_submission, leaderboard_matches_query, list_arenas, prism_arena_from_live, + prism_bpb_leaderboard, prism_submission, prism_telemetry, prism_window, + submission_matches_query, }; use site_types::coding_arena; use site_types::page_slice; @@ -438,7 +439,12 @@ async fn get_submissions( .and_then(Value::as_array) .cloned() .unwrap_or_default(); - let mut items: Vec<_> = rows.iter().filter_map(prism_submission).collect(); + // Public gallery: champions only (current top + Score>0 ex-tops). + let mut items: Vec<_> = rows + .iter() + .filter(|r| is_prism_champion_submission(r)) + .filter_map(prism_submission) + .collect(); if let Some(st_f) = status_filter { items.retain(|s| match st_f { "scored" => s.status == crate::SubmissionStatus::Scored, diff --git a/crates/site-data/src/map.rs b/crates/site-data/src/map.rs index 742904628..2fcc0af93 100644 --- a/crates/site-data/src/map.rs +++ b/crates/site-data/src/map.rs @@ -476,11 +476,25 @@ pub fn prism_submission(row: &Value) -> Option { }) } -/// Prism BPB leaderboard from terminal submissions (lower BPB ranks better). +/// True when a prism list/detail row is a public champion (Score>0 top / ex-top). +#[must_use] +pub fn is_prism_champion_submission(row: &Value) -> bool { + row.get("score") + .and_then(|s| { + if s.get("kind").and_then(Value::as_str) != Some("score") { + return Some(false); + } + Some(s.get("value").and_then(Value::as_u64)? > 0) + }) + .unwrap_or(false) +} + +/// Prism BPB leaderboard from **champion** terminal submissions (Score>0). /// -/// `elo` carries the BPB value so the existing leaderboard row contract can -/// surface rankings without inventing Elo/duels; `bpb` / `paramsM` mirror the -/// measured values explicitly for telemetry-aware clients. +/// Non-top submissions are hidden from the public board. `elo` carries the BPB +/// value so the existing leaderboard row contract can surface rankings without +/// inventing Elo/duels; `bpb` / `paramsM` mirror the measured values explicitly +/// for telemetry-aware clients. #[must_use] pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec { let mut best: HashMap)> = HashMap::new(); @@ -489,6 +503,9 @@ pub fn prism_bpb_leaderboard(subs: &[Value], epoch: u64) -> Vec if row.get("status").and_then(Value::as_str) != Some("terminated") { continue; } + if !is_prism_champion_submission(row) { + continue; + } let Some(bpb) = row.get("bpb").and_then(Value::as_f64) else { continue; }; @@ -1234,8 +1251,10 @@ mod tests { #[test] fn prism_leaderboard_exposes_bpb_and_params_fields() { let subs = vec![ - json!({"id":"a","status":"terminated","bpb":2.0,"miner_hotkey":"aa","n_params":12_000_000_u64}), - json!({"id":"b","status":"terminated","bpb":1.0,"miner_hotkey":"bb"}), + json!({"id":"a","status":"terminated","bpb":2.0,"miner_hotkey":"aa","n_params":12_000_000_u64,"score":{"kind":"score","value":100}}), + json!({"id":"b","status":"terminated","bpb":1.0,"miner_hotkey":"bb","score":{"kind":"score","value":200}}), + // Non-champion (Score 0) must stay off the public board. + json!({"id":"c","status":"terminated","bpb":0.5,"miner_hotkey":"cc","score":{"kind":"score","value":0}}), ]; let rows = prism_bpb_leaderboard(&subs, 3); assert_eq!(rows.len(), 2); @@ -1342,10 +1361,10 @@ mod tests { #[test] fn prism_bpb_leaderboard_ranks_lower_first() { let subs = vec![ - json!({"id":"a","status":"terminated","bpb":2.0,"miner_hotkey":"aa"}), - json!({"id":"b","status":"terminated","bpb":1.0,"miner_hotkey":"bb"}), - json!({"id":"c","status":"queued","bpb":0.1,"miner_hotkey":"cc"}), - json!({"id":"d","status":"terminated","bpb":0.5,"miner_hotkey":"aa"}), + json!({"id":"a","status":"terminated","bpb":2.0,"miner_hotkey":"aa","score":{"kind":"score","value":10}}), + json!({"id":"b","status":"terminated","bpb":1.0,"miner_hotkey":"bb","score":{"kind":"score","value":20}}), + json!({"id":"c","status":"queued","bpb":0.1,"miner_hotkey":"cc","score":{"kind":"score","value":30}}), + json!({"id":"d","status":"terminated","bpb":0.5,"miner_hotkey":"aa","score":{"kind":"score","value":40}}), ]; let rows = prism_bpb_leaderboard(&subs, 3); assert_eq!(rows.len(), 2); diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 85f869085..247ede930 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -106,7 +106,7 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. | | prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (10h grace + pre-reclaim log harvest), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | | prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. | -| prism LLM review | done | `prism-review` quality + similarity prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; anti-copy forces `Copied`/`Suspicious` → Score 0. | +| prism LLM review | done | `prism-review` quality + similarity-v3 prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; cheap `Copied` hard-zeros, cheap `Suspicious` advisory; generic LM tropes coerced; copy/similarity/agentic corpus = champions (Score>0) + baseline. | | prism API | done | Full status surface: submissions list/detail/events/status/jobs/recipe/baseline, idempotent accept. | | Phala / agent-v1 miner path | removed | External miners use HTTP submit only ([`external-miner/`](external-miner/)). | diff --git a/docs/PRISM.md b/docs/PRISM.md index b41d0def3..9b5e61e44 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -15,23 +15,24 @@ plus **training-only submissions** (`training.py` + a published `arch_id`) for the architecture competition (see below). Each evaluation is executed for real on a Lium GPU pod rented by the operator master (Sim backend in CI only). A **pre-LLM copy gate** rejects byte/AST copies of strictly-earlier -architectures (`created_at` ordered) without spending pod or LLM time. -The code is then LLM-reviewed for coherence, then judged for architecture -similarity (**`architecture.py` only** — `training.py` is exempt: the same -training script on two different architectures is legitimate), then run -through the shared **agentic** anti-cheat verifier (`challenge-agentic`: -tools + AST + metrics/receipt; OpenRouter when keyed, `SimAgent` in CI). -The LLM review also enforces the **telemetry contract**: `training.py` must -call `prism_telemetry.report(...)` + `prism_telemetry.finish_evaluation()`; -missing hooks are a hard contract violation (`missing_telemetry_hooks` → -`Score(0)`, terminal). Cheap review/similarity stay as first filters; -agentic is the primary anti-cheat judge. The LLM quality vote is a -**coherence gate, never a grader**: the final score is pure bpb, with -hard-zero on agentic `cheat`/`suspicious` and cheap `Copied`/`Suspicious`. -Missing agentic verdict is fail-closed (`ChallengeInternal`). Leaves are -D24-complete per chain epoch, emitted at epoch close from the finalized-since- -last-epoch batch (see **Leaf emission** below). Review -findings are audit events, not points. +**champion** architectures (Score>0 top + ex-tops; `created_at` ordered) +without spending pod or LLM time. The code is then LLM-reviewed for +coherence, then judged for architecture similarity (**`architecture.py` +only** — `training.py` is exempt: the same training script on two different +architectures is legitimate), then run through the shared **agentic** +anti-cheat verifier (`challenge-agentic`: tools + AST + metrics/receipt; +OpenRouter when keyed, `SimAgent` in CI). The LLM review also enforces the +**telemetry contract**: `training.py` must call `prism_telemetry.report(...)` ++ `prism_telemetry.finish_evaluation()`; missing hooks are a hard contract +violation (`missing_telemetry_hooks` → `Score(0)`, terminal). Cheap +`Copied` is a hard first filter; cheap `Suspicious` is advisory. Agentic is +the primary anti-cheat judge and must not treat standard LM components as +plagiarism. The LLM quality vote is a **coherence gate, never a grader**: +the final score is pure bpb, with hard-zero on agentic `cheat`/`suspicious` +and cheap `Copied`. Missing agentic verdict is fail-closed +(`ChallengeInternal`). Leaves are D24-complete per chain epoch, emitted at +epoch close from the finalized-since-last-epoch batch (see **Leaf emission** +below). Review findings are audit events, not points. This is **not** agent-challenge Phala/TDX attestation and **not** hypertraining B300 tournament code. @@ -181,28 +182,35 @@ absent/empty → publishing is a graceful no-op, scoring is unaffected. Before any pod rent, **pre-pod screens** (no GPU, no private eval assets) run in order and terminal-reject with `Score(0)` on hit: -1. **Pre-LLM copy gate** — candidate `architecture.py` vs recent submissions - from **other miners** (byte hash + `challenge-ast`; same hotkey/coldkey - prior art excluded). Byte/AST copy of a **strictly-earlier** submission is - rejected. Ties / unknown timestamps fall through; baseline is exempt. - Miners may probe this gate via `POST /v1/submissions/precheck` (quota - 3/coldkey/UTC day) without queuing a submission. +1. **Pre-LLM copy gate** — candidate `architecture.py` vs **champions** + (current top + historical Score>0 ex-tops) from **other miners** (byte hash + + `challenge-ast`; same hotkey/coldkey prior art excluded). Byte/AST copy + of a **strictly-earlier** champion is rejected. Ties / unknown timestamps + fall through; baseline is exempt. Miners may probe this gate via + `POST /v1/submissions/precheck` (quota 3/coldkey/UTC day) without queuing + a submission. 2. **Static source cheat** (`challenge_agentic::static_source_cheat`) — hardcoded `METRICS_JSON=` short-circuit; missing `prism_telemetry.report` / `finish_evaluation` hooks in `training.py`. -3. **Cheap AST similarity** (`prism-review`) — `Copied` / `Suspicious` hard - zero before rent. +3. **Cheap LLM similarity** (`prism-review` similarity-v3) — hard-zero on + `Copied` only before rent. `Suspicious` is advisory (does not reject). + Parsers coerce verdicts whose evidence is only standard LM components + (RMSNorm / RoPE / SwiGLU / LayerNorm / gated or parallel residual, …). After measure, the LLM quality review and the shared `challenge-agentic` loop inspect sources + metrics/receipt with read-only tools (`list_dir`, `read_file`, `ast_summary`, `ast_diff_nearest`, `read_metrics`) against an -**architecture-only** corpus of baseline + other miners' recent submissions. -Final judge is the mandatory `submit_verdict` function-call. +**architecture-only** corpus of baseline + champions. Final judge is the +mandatory `submit_verdict` function-call. Agentic must not treat generic +modern-LM components as plagiarism; AST bands (`≥8500` suspicious / +`≥9500` cheat) remain the structural copy thresholds. | Verdict | Leaf effect | |---------|-------------| | `clean` | proceed; score = pure bpb on `[0, SCORE_MAX]` | -| `suspicious` / `cheat` | `Score(0)` via `combine_final` | +| agentic `suspicious` / `cheat` | `Score(0)` via `combine_final` | +| cheap LLM `Copied` | `Score(0)` | +| cheap LLM `Suspicious` | advisory only (not a hard zero) | | missing / unparseable | `NoScore(ChallengeInternal)` (fail-closed) | Cheat taxonomy (Prism-relevant): @@ -215,9 +223,11 @@ Cheat taxonomy (Prism-relevant): | `near_identical_harness_copy` | Near-identical corpus copy | | `missing_telemetry_hooks` | `training.py` does not call `prism_telemetry.report` + `finish_evaluation` | -Cheap `Copied` / `Suspicious` from single-shot similarity remain hard-zero -first filters; agentic is the **primary** anti-cheat judge. LLM quality stays -audit-only for the bpb score (coherence gate, never a grader). +Cheap `Copied` from single-shot similarity remains a hard-zero first filter; +cheap `Suspicious` is advisory. Agentic is the **primary** anti-cheat judge. +Public site gallery/leaderboard list **champions only** (Score>0); operators +still see the full corpus via the challenge API. LLM quality stays audit-only +for the bpb score (coherence gate, never a grader). ## Crates diff --git a/docs/PRISM_RECIPE.md b/docs/PRISM_RECIPE.md index 6b7ab3746..9cf3aaf7a 100644 --- a/docs/PRISM_RECIPE.md +++ b/docs/PRISM_RECIPE.md @@ -126,25 +126,26 @@ verdict, quality notes and issues are kept as audit records review still gates eligibility: - similarity verdict `Copied` → hard **Score 0** -- similarity verdict `Suspicious` → hard **Score 0** until reviewed +- similarity verdict `Suspicious` → advisory only (not a hard zero; agentic is the judge) - harness/antipattern failure → `ChallengeInternal` maps to `NoScore` reason ## Anti-copy review A **pre-LLM copy gate** first compares the candidate `architecture.py` -against recent submissions from **other miners** (byte hash + AST -fingerprints, `created_at` ordered; same-`miner_hotkey` and -same-`miner_coldkey` prior art excluded): a byte/AST copy of a -strictly-earlier architecture is terminal `rejected` with zero score — no pod -time, no LLM spend. The baseline is exempt (everyone may start from it); -created_at ties fall through to the LLM path below. +against **champion** submissions (Score>0 current top + historical ex-tops) +from **other miners** (byte hash + AST fingerprints, `created_at` ordered; +same-`miner_hotkey` and same-`miner_coldkey` prior art excluded): a byte/AST +copy of a strictly-earlier champion architecture is terminal `rejected` with +zero score — no pod time, no LLM spend. The baseline is exempt (everyone may +start from it); created_at ties fall through to the LLM path below. Each remaining submission then faces an LLM review on the master (`OpenRouter` when the key file `/run/base/openrouter/api_key` exists, else the deterministic `SimReviewer`) over its **architecture only** vs. the -recipe **baseline plus earlier other-miner submissions** (`prism_submission` -history, capped at the 6 most recent records; same hotkey/coldkey exclusion). -Since similarity v2, `training.py` is exempt from both candidate and corpus: -the same training script on two different architectures is legitimate. -Verdicts: `Original` / `Suspicious` / `Copied`, with a similarity score and -evidence line — all stored append-only in `prism_stage_event`. +recipe **baseline plus champions** (capped; same hotkey/coldkey exclusion). +Since similarity v2/v3, `training.py` is exempt from both candidate and +corpus: the same training script on two different architectures is +legitimate. Verdicts: `Original` / `Suspicious` / `Copied`, with a similarity +score and evidence line — all stored append-only in `prism_stage_event`. +Generic modern-LM components (RMSNorm, RoPE, SwiGLU, …) must not appear as +copy evidence; parsers coerce those false positives to `Original`. diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 866f903c8..433eb176d 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -34,7 +34,10 @@ no step curve is stored. `PrismWindow.tokenBudget` is **0** unless a recipe publishes a fixed token quota (prism ≥1.2 does not — caps are wall-clock / steps / params). Chart x-values still come from miner telemetry (`layer_stats.tokens` when present); clients must not label the max observed -x as an egalitarian “token window.” +x as an egalitarian “token window.” Prism public submissions + BPB +leaderboard list **champions only** (`score.kind=score` and `value > 0` — +current top and historical ex-tops); non-top rows stay on the operator +challenge API. `GET /v1/site/arenas/{slug}/submissions` and `/leaderboard` accept optional `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle, diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 7189eef8d..739d48b00 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -130,11 +130,14 @@ submission and never rents a Lium pod. ## Scoring (summary) Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`. -Cheap similarity plus the shared **agentic** gate (AST + metrics/receipt) force -hard-zero on `cheat` / `suspicious` (and cheap `Copied` / `Suspicious`). -Copy/similarity corpora exclude your own prior art (same hotkey **or** same -coldkey), so iterating via a new hotkey under the same coldkey is not treated -as a cross-miner copy. LLM quality is coherence-only, not a grader. +The shared **agentic** gate (AST + metrics/receipt) hard-zeros `cheat` / +`suspicious`. Cheap LLM similarity hard-zeros only `Copied` (`Suspicious` is +advisory). Copy/similarity corpora are **champions only** (current top + +historical Score>0 ex-tops) plus baseline — not every past submission — and +still exclude your own prior art (same hotkey **or** same coldkey). Standard +components (RMSNorm, RoPE, SwiGLU, LayerNorm, gated/parallel residual, …) are +**not** plagiarism signals. LLM quality is coherence-only, not a grader. +Public gallery/leaderboard show champions only. **Competition:** per epoch you are credited the max of (a) your own best training result and (b) for each arch you own, that arch's best result by *any* trainer — architecture owners are rewarded