From 2e9a1c260a189c403767ef9f01c2041cba834f9d Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:56:37 +0000 Subject: [PATCH] fix(prism): threshold cheap Suspicious in combine_final Score(0) only for Copied or Suspicious at/above 0.9 with non-trope evidence so 0.7 RMSNorm/SwiGLU false positives no longer wipe leaves. --- crates/prism-challenge/src/orchestrator.rs | 19 +++-- crates/prism-challenge/src/score.rs | 90 ++++++++++++++++------ crates/prism-review/src/generic_arch.rs | 58 +++++++++++++- crates/prism-review/src/lib.rs | 5 +- docs/COMPLETENESS.md | 2 +- docs/PRISM.md | 28 ++++--- docs/PRISM_RECIPE.md | 4 +- docs/external-miner/prism.md | 14 ++-- docs/external-miner/troubleshoot.md | 2 +- 9 files changed, 171 insertions(+), 51 deletions(-) diff --git a/crates/prism-challenge/src/orchestrator.rs b/crates/prism-challenge/src/orchestrator.rs index 7466e2d42..13b9ce478 100644 --- a/crates/prism-challenge/src/orchestrator.rs +++ b/crates/prism-challenge/src/orchestrator.rs @@ -390,6 +390,8 @@ impl Orchestrator { bpb: b, quality: review.quality_score, similarity: similarity.kind, + similarity_score: similarity.score, + similarity_evidence: similarity.evidence.clone(), agentic: agentic.verdict, }, None => FinalOutcome::ChallengeInternal, @@ -457,11 +459,18 @@ impl Orchestrator { return None; } }; - // Only hard-reject LLM `Copied`. `Suspicious` is advisory — agentic - // (post-pod, AST-thresholded) is the primary judge. Generic LM tropes - // are coerced to Original in prism-review parsers. - if matches!(similarity.kind, prism_review::SimilarityKind::Copied) { - let detail = format!("pre-pod similarity: {:?}", similarity.kind); + // Hard-reject LLM `Copied`, and high-confidence `Suspicious` + // (score ≥ 0.9 with non-trope evidence). Below-threshold / trope-only + // Suspicious is not a wipe — parser coercion + combine_final agree. + if prism_review::cheap_similarity_hard_zeros( + similarity.kind, + similarity.score, + &similarity.evidence, + ) { + let detail = format!( + "pre-pod similarity: {:?} score={:.2}", + similarity.kind, similarity.score + ); self.reject_pre_pod(row, Some(similarity), None, detail) .await; return None; diff --git a/crates/prism-challenge/src/score.rs b/crates/prism-challenge/src/score.rs index ac2811204..6d9075403 100644 --- a/crates/prism-challenge/src/score.rs +++ b/crates/prism-challenge/src/score.rs @@ -6,6 +6,7 @@ use bundle::NoScoreReasonCode; use prism_pipeline::score_from_bpb; +use prism_review::cheap_similarity_hard_zeros; /// Final outcomes after LLM review + similarity + agentic gates (orchestrator v2). #[derive(Debug, Clone, PartialEq)] @@ -18,6 +19,11 @@ pub enum FinalOutcome { quality: u16, /// Cheap single-shot similarity class. similarity: prism_review::SimilarityKind, + /// Cheap LLM similarity confidence `0.0..1.0` (compared for + /// `Suspicious`; see [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`]). + similarity_score: f64, + /// Cheap LLM evidence lines (trope-only → no Score wipe). + similarity_evidence: Vec, /// Agentic anti-cheat verdict (primary gate). agentic: challenge_agentic::VerdictKind, }, @@ -29,10 +35,17 @@ pub enum FinalOutcome { /// /// The score is **pure bpb**: the LLM review is an anti-cheat / coherence /// GATE, never a grader — its quality vote and issues are recorded as audit -/// events but never add nor remove points. Agentic `Cheat`/`Suspicious` and -/// cheap similarity `Copied` are hard gates (miner-attributable `Score{0}`). -/// Cheap LLM `Suspicious` is advisory only (agentic + AST thresholds are the -/// primary judge). Missing agentic verdict is fail-closed upstream as +/// events but never add nor remove points. +/// +/// Hard gates (miner-attributable `Score{0}`): +/// - Agentic `Cheat` / `Suspicious` (AST bands already applied upstream). +/// - Cheap LLM `Copied`. +/// - Cheap LLM `Suspicious` **only** when +/// `similarity_score >=` [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`] +/// `(0.9)` **and** evidence is not generic-trope-only (RMSNorm / SwiGLU / +/// LayerNorm / …). Below the threshold (e.g. 0.7 tropes) → no score wipe. +/// +/// Missing agentic verdict is fail-closed upstream as /// [`FinalOutcome::ChallengeInternal`]. /// /// # Panics @@ -49,12 +62,14 @@ pub fn combine_final(outcome: &FinalOutcome) -> prism_store::FinalScore { bpb, quality: _, similarity, + similarity_score, + similarity_evidence, agentic, } => { if matches!(agentic, VerdictKind::Cheat | VerdictKind::Suspicious) { return FinalScore::Score(0); } - if matches!(similarity, prism_review::SimilarityKind::Copied) { + if cheap_similarity_hard_zeros(*similarity, *similarity_score, similarity_evidence) { return FinalScore::Score(0); } FinalScore::Score(score_from_bpb(*bpb)) @@ -70,39 +85,60 @@ mod final_tests { use prism_review::SimilarityKind::Original; use prism_review::SimilarityKind::Suspicious; - #[test] - fn copied_is_hard_zero() { - let o = FinalOutcome::Measured { + fn measured( + similarity: prism_review::SimilarityKind, + similarity_score: f64, + evidence: &[&str], + agentic: challenge_agentic::VerdictKind, + ) -> FinalOutcome { + FinalOutcome::Measured { bpb: 1.0, quality: 900, - similarity: Copied, - agentic: challenge_agentic::VerdictKind::Clean, - }; + similarity, + similarity_score, + similarity_evidence: evidence.iter().map(|s| (*s).to_owned()).collect(), + agentic, + } + } + + #[test] + fn copied_is_hard_zero() { + let o = measured(Copied, 0.5, &[], challenge_agentic::VerdictKind::Clean); assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0)); } #[test] - fn cheap_suspicious_is_not_hard_zero() { - let o = FinalOutcome::Measured { - bpb: 1.0, - quality: 900, - similarity: Suspicious, - agentic: challenge_agentic::VerdictKind::Clean, - }; + fn suspicious_0_7_tropes_not_hard_zero() { + let o = measured( + Suspicious, + 0.7, + &[ + "RMSNorm usage", + "SwiGLU feed-forward", + "Layer normalization", + ], + challenge_agentic::VerdictKind::Clean, + ); assert!(matches!( combine_final(&o), prism_store::FinalScore::Score(v) if v > 0 )); } + #[test] + fn suspicious_0_99_real_copy_is_hard_zero() { + let o = measured( + Suspicious, + 0.99, + &["same custom DualPathBlock wiring as subm:aabbccdd"], + challenge_agentic::VerdictKind::Clean, + ); + assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0)); + } + #[test] fn agentic_cheat_is_hard_zero() { - let o = FinalOutcome::Measured { - bpb: 1.0, - quality: 900, - similarity: Original, - agentic: challenge_agentic::VerdictKind::Cheat, - }; + let o = measured(Original, 0.0, &[], challenge_agentic::VerdictKind::Cheat); assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0)); } @@ -114,12 +150,16 @@ mod final_tests { bpb: 0.5, quality: 900, similarity: Original, + similarity_score: 0.0, + similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, }; let lo_same_bpb = FinalOutcome::Measured { bpb: 0.5, quality: 0, similarity: Original, + similarity_score: 0.0, + similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, }; assert_eq!(combine_final(&hi), combine_final(&lo_same_bpb)); @@ -128,6 +168,8 @@ mod final_tests { bpb: 4.0, quality: 1000, similarity: Original, + similarity_score: 0.0, + similarity_evidence: vec![], agentic: challenge_agentic::VerdictKind::Clean, }; match (combine_final(&hi), combine_final(&worse_bpb)) { diff --git a/crates/prism-review/src/generic_arch.rs b/crates/prism-review/src/generic_arch.rs index 80a53a2d5..8b6b09f0e 100644 --- a/crates/prism-review/src/generic_arch.rs +++ b/crates/prism-review/src/generic_arch.rs @@ -2,6 +2,11 @@ use crate::types::{SimilarityKind, SimilarityVerdict}; +/// Cheap LLM `Suspicious` hard-zeros at/above this confidence when evidence +/// is not generic-trope-only. Shared by parse coercion, pre-pod reject, and +/// [`cheap_similarity_hard_zeros`] / `combine_final`. +pub const SUSPICIOUS_HARD_ZERO_THRESHOLD: f64 = 0.9; + /// Substrings that are standard modern-LM vocabulary (case-insensitive). /// When *every* evidence line matches one of these (and nothing else /// substantive), a `suspicious` / soft-`copied` verdict is coerced to @@ -50,6 +55,22 @@ pub fn evidence_is_only_generic_tropes(evidence: &[String]) -> bool { }) } +/// Whether cheap LLM similarity should wipe the leaf (`Score(0)`). +/// +/// `Copied` always wipes. `Suspicious` wipes only when `score >=` +/// [`SUSPICIOUS_HARD_ZERO_THRESHOLD`] and evidence is not generic-trope-only +/// (`RMSNorm` / `SwiGLU` / `LayerNorm` / …). `Original` never wipes. +#[must_use] +pub fn cheap_similarity_hard_zeros(kind: SimilarityKind, score: f64, evidence: &[String]) -> bool { + match kind { + SimilarityKind::Copied => true, + SimilarityKind::Suspicious => { + score >= SUSPICIOUS_HARD_ZERO_THRESHOLD && !evidence_is_only_generic_tropes(evidence) + } + SimilarityKind::Original => false, + } +} + /// Coerce false-positive LLM similarity verdicts that cite only standard /// components. Hard `copied` with score ≥ 0.95 is left alone (near-verbatim /// copies can still mention shared blocks in evidence). @@ -57,7 +78,7 @@ pub fn evidence_is_only_generic_tropes(evidence: &[String]) -> bool { pub fn coerce_generic_similarity(mut v: SimilarityVerdict) -> SimilarityVerdict { let only_generic = evidence_is_only_generic_tropes(&v.evidence); match v.kind { - SimilarityKind::Suspicious if only_generic || v.score < 0.9 => { + SimilarityKind::Suspicious if only_generic || v.score < SUSPICIOUS_HARD_ZERO_THRESHOLD => { v.kind = SimilarityKind::Original; if only_generic { v.evidence.insert( @@ -133,4 +154,39 @@ mod tests { )); assert!(matches!(v.kind, SimilarityKind::Copied)); } + + #[test] + fn hard_zeros_threshold_on_suspicious() { + let tropes = [ + "RMSNorm usage".into(), + "SwiGLU feed-forward".into(), + "Layer normalization".into(), + ]; + assert!(!cheap_similarity_hard_zeros( + SimilarityKind::Suspicious, + 0.7, + &tropes + )); + assert!(!cheap_similarity_hard_zeros( + SimilarityKind::Suspicious, + 0.99, + &tropes + )); + let real = ["same custom DualPathBlock wiring as subm:aabbccdd".into()]; + assert!(!cheap_similarity_hard_zeros( + SimilarityKind::Suspicious, + 0.7, + &real + )); + assert!(cheap_similarity_hard_zeros( + SimilarityKind::Suspicious, + 0.99, + &real + )); + assert!(cheap_similarity_hard_zeros( + SimilarityKind::Copied, + 0.5, + &[] + )); + } } diff --git a/crates/prism-review/src/lib.rs b/crates/prism-review/src/lib.rs index 03605348c..f151a638a 100644 --- a/crates/prism-review/src/lib.rs +++ b/crates/prism-review/src/lib.rs @@ -22,7 +22,10 @@ mod prompts; mod sim; mod types; -pub use generic_arch::{coerce_generic_similarity, evidence_is_only_generic_tropes}; +pub use generic_arch::{ + cheap_similarity_hard_zeros, coerce_generic_similarity, evidence_is_only_generic_tropes, + SUSPICIOUS_HARD_ZERO_THRESHOLD, +}; pub use llm::{load_api_key_file, OpenRouterClient}; pub use prompts::{REVIEW_PROMPT_VERSION, SIMILARITY_PROMPT_VERSION}; pub use sim::SimReviewer; diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index 247ede930..1bcc37196 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -106,7 +106,7 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. | | prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (10h grace + pre-reclaim log harvest), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. | | prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. | -| prism LLM review | done | `prism-review` quality + similarity-v3 prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; cheap `Copied` hard-zeros, cheap `Suspicious` advisory; generic LM tropes coerced; copy/similarity/agentic corpus = champions (Score>0) + baseline. | +| prism LLM review | done | `prism-review` quality + similarity-v3 prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; cheap `Copied` hard-zeros; cheap `Suspicious` hard-zeros only at `score ≥ 0.9` with non-trope evidence (`SUSPICIOUS_HARD_ZERO_THRESHOLD`); generic LM tropes coerced; copy/similarity/agentic corpus = champions (Score>0) + baseline. | | prism API | done | Full status surface: submissions list/detail/events/status/jobs/recipe/baseline, idempotent accept. | | Phala / agent-v1 miner path | removed | External miners use HTTP submit only ([`external-miner/`](external-miner/)). | diff --git a/docs/PRISM.md b/docs/PRISM.md index 9b5e61e44..a5319766a 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -25,12 +25,14 @@ OpenRouter when keyed, `SimAgent` in CI). The LLM review also enforces the **telemetry contract**: `training.py` must call `prism_telemetry.report(...)` + `prism_telemetry.finish_evaluation()`; missing hooks are a hard contract violation (`missing_telemetry_hooks` → `Score(0)`, terminal). Cheap -`Copied` is a hard first filter; cheap `Suspicious` is advisory. Agentic is -the primary anti-cheat judge and must not treat standard LM components as -plagiarism. The LLM quality vote is a **coherence gate, never a grader**: -the final score is pure bpb, with hard-zero on agentic `cheat`/`suspicious` -and cheap `Copied`. Missing agentic verdict is fail-closed -(`ChallengeInternal`). Leaves are D24-complete per chain epoch, emitted at +`Copied` is a hard first filter; cheap `Suspicious` hard-zeros only when +`score ≥ 0.9` (`SUSPICIOUS_HARD_ZERO_THRESHOLD`) and evidence is not +generic-trope-only. Agentic is the primary anti-cheat judge and must not +treat standard LM components as plagiarism. The LLM quality vote is a +**coherence gate, never a grader**: the final score is pure bpb, with +hard-zero on agentic `cheat`/`suspicious` and cheap `Copied` / high-confidence +`Suspicious`. Missing agentic verdict is fail-closed (`ChallengeInternal`). +Leaves are D24-complete per chain epoch, emitted at epoch close from the finalized-since-last-epoch batch (see **Leaf emission** below). Review findings are audit events, not points. @@ -193,9 +195,11 @@ in order and terminal-reject with `Score(0)` on hit: hardcoded `METRICS_JSON=` short-circuit; missing `prism_telemetry.report` / `finish_evaluation` hooks in `training.py`. 3. **Cheap LLM similarity** (`prism-review` similarity-v3) — hard-zero on - `Copied` only before rent. `Suspicious` is advisory (does not reject). - Parsers coerce verdicts whose evidence is only standard LM components - (RMSNorm / RoPE / SwiGLU / LayerNorm / gated or parallel residual, …). + `Copied`, and on `Suspicious` when `score ≥ 0.9` with non-trope evidence + (`combine_final` + pre-pod share [`cheap_similarity_hard_zeros`]). + Below-threshold `Suspicious` (e.g. 0.7) does not wipe. Parsers coerce + verdicts whose evidence is only standard LM components (RMSNorm / RoPE / + SwiGLU / LayerNorm / gated or parallel residual, …). After measure, the LLM quality review and the shared `challenge-agentic` loop inspect sources + metrics/receipt with read-only tools (`list_dir`, @@ -210,7 +214,7 @@ modern-LM components as plagiarism; AST bands (`≥8500` suspicious / | `clean` | proceed; score = pure bpb on `[0, SCORE_MAX]` | | agentic `suspicious` / `cheat` | `Score(0)` via `combine_final` | | cheap LLM `Copied` | `Score(0)` | -| cheap LLM `Suspicious` | advisory only (not a hard zero) | +| cheap LLM `Suspicious` | `Score(0)` iff `score ≥ 0.9` and evidence not trope-only; else no wipe | | missing / unparseable | `NoScore(ChallengeInternal)` (fail-closed) | Cheat taxonomy (Prism-relevant): @@ -224,7 +228,9 @@ Cheat taxonomy (Prism-relevant): | `missing_telemetry_hooks` | `training.py` does not call `prism_telemetry.report` + `finish_evaluation` | Cheap `Copied` from single-shot similarity remains a hard-zero first filter; -cheap `Suspicious` is advisory. Agentic is the **primary** anti-cheat judge. +cheap `Suspicious` uses the numeric score against +`SUSPICIOUS_HARD_ZERO_THRESHOLD` (0.9) plus trope coercion. Agentic is the +**primary** anti-cheat judge. Public site gallery/leaderboard list **champions only** (Score>0); operators still see the full corpus via the challenge API. LLM quality stays audit-only for the bpb score (coherence gate, never a grader). diff --git a/docs/PRISM_RECIPE.md b/docs/PRISM_RECIPE.md index 9cf3aaf7a..a2664a78a 100644 --- a/docs/PRISM_RECIPE.md +++ b/docs/PRISM_RECIPE.md @@ -126,7 +126,9 @@ verdict, quality notes and issues are kept as audit records review still gates eligibility: - similarity verdict `Copied` → hard **Score 0** -- similarity verdict `Suspicious` → advisory only (not a hard zero; agentic is the judge) +- similarity verdict `Suspicious` → **Score 0** only when `score ≥ 0.9` and + evidence is not generic-trope-only (else no wipe; agentic remains the + structural judge) - harness/antipattern failure → `ChallengeInternal` maps to `NoScore` reason ## Anti-copy review diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 739d48b00..11c5f37b2 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -131,12 +131,14 @@ submission and never rents a Lium pod. Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`. The shared **agentic** gate (AST + metrics/receipt) hard-zeros `cheat` / -`suspicious`. Cheap LLM similarity hard-zeros only `Copied` (`Suspicious` is -advisory). Copy/similarity corpora are **champions only** (current top + -historical Score>0 ex-tops) plus baseline — not every past submission — and -still exclude your own prior art (same hotkey **or** same coldkey). Standard -components (RMSNorm, RoPE, SwiGLU, LayerNorm, gated/parallel residual, …) are -**not** plagiarism signals. LLM quality is coherence-only, not a grader. +`suspicious`. Cheap LLM similarity hard-zeros `Copied`, and `Suspicious` only +when confidence `≥ 0.9` with non-generic evidence (below that — e.g. 0.7 citing +RMSNorm/SwiGLU/LayerNorm — does **not** wipe your score). Copy/similarity +corpora are **champions only** (current top + historical Score>0 ex-tops) plus +baseline — not every past submission — and still exclude your own prior art +(same hotkey **or** same coldkey). Standard components (RMSNorm, RoPE, SwiGLU, +LayerNorm, gated/parallel residual, …) are **not** plagiarism signals. LLM +quality is coherence-only, not a grader. Public gallery/leaderboard show champions only. **Competition:** per epoch you are credited the max of (a) your own best training result and (b) for each arch you diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 8695a30bc..2d956bac9 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -22,7 +22,7 @@ | Symptom | Likely cause | What to check | |---------|--------------|---------------| | Rejected submit | Recipe contract | `GET /v1/recipe` + baseline; follow [`PRISM_RECIPE.md`](../PRISM_RECIPE.md) | -| Score 0 after review | `Copied` / `Suspicious` | Similarity gate; rewrite; do not paste baseline wholesale | +| Score 0 after review | `Copied` / high-confidence `Suspicious` (≥0.9, non-trope) | Similarity gate; rewrite unique structure; tropes alone are not plagiarism | | `similar: true` on precheck | Would hit intake copy gate | Rewrite `architecture.py`; baseline is fine to start from | | `429 precheck_quota_exceeded` | 3 prechecks/coldkey/UTC day used | Wait until next UTC day; rotating hotkeys does not reset | | Stuck `Provisioning` | Lium market thinness | Ops-side; watch `GET /v1/jobs` / events |