Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions crates/prism-challenge/src/orchestrator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,8 @@ impl<C: ChainClient + Send> Orchestrator<C> {
bpb: b,
quality: review.quality_score,
similarity: similarity.kind,
similarity_score: similarity.score,
similarity_evidence: similarity.evidence.clone(),
agentic: agentic.verdict,
},
None => FinalOutcome::ChallengeInternal,
Expand Down Expand Up @@ -457,11 +459,18 @@ impl<C: ChainClient + Send> Orchestrator<C> {
return None;
}
};
// Only hard-reject LLM `Copied`. `Suspicious` is advisory — agentic
// (post-pod, AST-thresholded) is the primary judge. Generic LM tropes
// are coerced to Original in prism-review parsers.
if matches!(similarity.kind, prism_review::SimilarityKind::Copied) {
let detail = format!("pre-pod similarity: {:?}", similarity.kind);
// Hard-reject LLM `Copied`, and high-confidence `Suspicious`
// (score ≥ 0.9 with non-trope evidence). Below-threshold / trope-only
// Suspicious is not a wipe — parser coercion + combine_final agree.
if prism_review::cheap_similarity_hard_zeros(
similarity.kind,
similarity.score,
&similarity.evidence,
) {
let detail = format!(
"pre-pod similarity: {:?} score={:.2}",
similarity.kind, similarity.score
);
self.reject_pre_pod(row, Some(similarity), None, detail)
.await;
return None;
Expand Down
90 changes: 66 additions & 24 deletions crates/prism-challenge/src/score.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

use bundle::NoScoreReasonCode;
use prism_pipeline::score_from_bpb;
use prism_review::cheap_similarity_hard_zeros;

/// Final outcomes after LLM review + similarity + agentic gates (orchestrator v2).
#[derive(Debug, Clone, PartialEq)]
Expand All @@ -18,6 +19,11 @@ pub enum FinalOutcome {
quality: u16,
/// Cheap single-shot similarity class.
similarity: prism_review::SimilarityKind,
/// Cheap LLM similarity confidence `0.0..1.0` (compared for
/// `Suspicious`; see [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`]).
similarity_score: f64,
/// Cheap LLM evidence lines (trope-only → no Score wipe).
similarity_evidence: Vec<String>,
/// Agentic anti-cheat verdict (primary gate).
agentic: challenge_agentic::VerdictKind,
},
Expand All @@ -29,10 +35,17 @@ pub enum FinalOutcome {
///
/// The score is **pure bpb**: the LLM review is an anti-cheat / coherence
/// GATE, never a grader — its quality vote and issues are recorded as audit
/// events but never add nor remove points. Agentic `Cheat`/`Suspicious` and
/// cheap similarity `Copied` are hard gates (miner-attributable `Score{0}`).
/// Cheap LLM `Suspicious` is advisory only (agentic + AST thresholds are the
/// primary judge). Missing agentic verdict is fail-closed upstream as
/// events but never add nor remove points.
///
/// Hard gates (miner-attributable `Score{0}`):
/// - Agentic `Cheat` / `Suspicious` (AST bands already applied upstream).
/// - Cheap LLM `Copied`.
/// - Cheap LLM `Suspicious` **only** when
/// `similarity_score >=` [`prism_review::SUSPICIOUS_HARD_ZERO_THRESHOLD`]
/// `(0.9)` **and** evidence is not generic-trope-only (RMSNorm / SwiGLU /
/// LayerNorm / …). Below the threshold (e.g. 0.7 tropes) → no score wipe.
///
/// Missing agentic verdict is fail-closed upstream as
/// [`FinalOutcome::ChallengeInternal`].
///
/// # Panics
Expand All @@ -49,12 +62,14 @@ pub fn combine_final(outcome: &FinalOutcome) -> prism_store::FinalScore {
bpb,
quality: _,
similarity,
similarity_score,
similarity_evidence,
agentic,
} => {
if matches!(agentic, VerdictKind::Cheat | VerdictKind::Suspicious) {
return FinalScore::Score(0);
}
if matches!(similarity, prism_review::SimilarityKind::Copied) {
if cheap_similarity_hard_zeros(*similarity, *similarity_score, similarity_evidence) {
return FinalScore::Score(0);
}
FinalScore::Score(score_from_bpb(*bpb))
Expand All @@ -70,39 +85,60 @@ mod final_tests {
use prism_review::SimilarityKind::Original;
use prism_review::SimilarityKind::Suspicious;

#[test]
fn copied_is_hard_zero() {
let o = FinalOutcome::Measured {
fn measured(
similarity: prism_review::SimilarityKind,
similarity_score: f64,
evidence: &[&str],
agentic: challenge_agentic::VerdictKind,
) -> FinalOutcome {
FinalOutcome::Measured {
bpb: 1.0,
quality: 900,
similarity: Copied,
agentic: challenge_agentic::VerdictKind::Clean,
};
similarity,
similarity_score,
similarity_evidence: evidence.iter().map(|s| (*s).to_owned()).collect(),
agentic,
}
}

#[test]
fn copied_is_hard_zero() {
let o = measured(Copied, 0.5, &[], challenge_agentic::VerdictKind::Clean);
assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0));
}

#[test]
fn cheap_suspicious_is_not_hard_zero() {
let o = FinalOutcome::Measured {
bpb: 1.0,
quality: 900,
similarity: Suspicious,
agentic: challenge_agentic::VerdictKind::Clean,
};
fn suspicious_0_7_tropes_not_hard_zero() {
let o = measured(
Suspicious,
0.7,
&[
"RMSNorm usage",
"SwiGLU feed-forward",
"Layer normalization",
],
challenge_agentic::VerdictKind::Clean,
);
assert!(matches!(
combine_final(&o),
prism_store::FinalScore::Score(v) if v > 0
));
}

#[test]
fn suspicious_0_99_real_copy_is_hard_zero() {
let o = measured(
Suspicious,
0.99,
&["same custom DualPathBlock wiring as subm:aabbccdd"],
challenge_agentic::VerdictKind::Clean,
);
assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0));
}

#[test]
fn agentic_cheat_is_hard_zero() {
let o = FinalOutcome::Measured {
bpb: 1.0,
quality: 900,
similarity: Original,
agentic: challenge_agentic::VerdictKind::Cheat,
};
let o = measured(Original, 0.0, &[], challenge_agentic::VerdictKind::Cheat);
assert_eq!(combine_final(&o), prism_store::FinalScore::Score(0));
}

Expand All @@ -114,12 +150,16 @@ mod final_tests {
bpb: 0.5,
quality: 900,
similarity: Original,
similarity_score: 0.0,
similarity_evidence: vec![],
agentic: challenge_agentic::VerdictKind::Clean,
};
let lo_same_bpb = FinalOutcome::Measured {
bpb: 0.5,
quality: 0,
similarity: Original,
similarity_score: 0.0,
similarity_evidence: vec![],
agentic: challenge_agentic::VerdictKind::Clean,
};
assert_eq!(combine_final(&hi), combine_final(&lo_same_bpb));
Expand All @@ -128,6 +168,8 @@ mod final_tests {
bpb: 4.0,
quality: 1000,
similarity: Original,
similarity_score: 0.0,
similarity_evidence: vec![],
agentic: challenge_agentic::VerdictKind::Clean,
};
match (combine_final(&hi), combine_final(&worse_bpb)) {
Expand Down
58 changes: 57 additions & 1 deletion crates/prism-review/src/generic_arch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@

use crate::types::{SimilarityKind, SimilarityVerdict};

/// Cheap LLM `Suspicious` hard-zeros at/above this confidence when evidence
/// is not generic-trope-only. Shared by parse coercion, pre-pod reject, and
/// [`cheap_similarity_hard_zeros`] / `combine_final`.
pub const SUSPICIOUS_HARD_ZERO_THRESHOLD: f64 = 0.9;

/// Substrings that are standard modern-LM vocabulary (case-insensitive).
/// When *every* evidence line matches one of these (and nothing else
/// substantive), a `suspicious` / soft-`copied` verdict is coerced to
Expand Down Expand Up @@ -50,14 +55,30 @@ pub fn evidence_is_only_generic_tropes(evidence: &[String]) -> bool {
})
}

/// Whether cheap LLM similarity should wipe the leaf (`Score(0)`).
///
/// `Copied` always wipes. `Suspicious` wipes only when `score >=`
/// [`SUSPICIOUS_HARD_ZERO_THRESHOLD`] and evidence is not generic-trope-only
/// (`RMSNorm` / `SwiGLU` / `LayerNorm` / …). `Original` never wipes.
#[must_use]
pub fn cheap_similarity_hard_zeros(kind: SimilarityKind, score: f64, evidence: &[String]) -> bool {
match kind {
SimilarityKind::Copied => true,
SimilarityKind::Suspicious => {
score >= SUSPICIOUS_HARD_ZERO_THRESHOLD && !evidence_is_only_generic_tropes(evidence)
}
SimilarityKind::Original => false,
}
}

/// Coerce false-positive LLM similarity verdicts that cite only standard
/// components. Hard `copied` with score ≥ 0.95 is left alone (near-verbatim
/// copies can still mention shared blocks in evidence).
#[must_use]
pub fn coerce_generic_similarity(mut v: SimilarityVerdict) -> SimilarityVerdict {
let only_generic = evidence_is_only_generic_tropes(&v.evidence);
match v.kind {
SimilarityKind::Suspicious if only_generic || v.score < 0.9 => {
SimilarityKind::Suspicious if only_generic || v.score < SUSPICIOUS_HARD_ZERO_THRESHOLD => {
v.kind = SimilarityKind::Original;
if only_generic {
v.evidence.insert(
Expand Down Expand Up @@ -133,4 +154,39 @@ mod tests {
));
assert!(matches!(v.kind, SimilarityKind::Copied));
}

#[test]
fn hard_zeros_threshold_on_suspicious() {
let tropes = [
"RMSNorm usage".into(),
"SwiGLU feed-forward".into(),
"Layer normalization".into(),
];
assert!(!cheap_similarity_hard_zeros(
SimilarityKind::Suspicious,
0.7,
&tropes
));
assert!(!cheap_similarity_hard_zeros(
SimilarityKind::Suspicious,
0.99,
&tropes
));
let real = ["same custom DualPathBlock wiring as subm:aabbccdd".into()];
assert!(!cheap_similarity_hard_zeros(
SimilarityKind::Suspicious,
0.7,
&real
));
assert!(cheap_similarity_hard_zeros(
SimilarityKind::Suspicious,
0.99,
&real
));
assert!(cheap_similarity_hard_zeros(
SimilarityKind::Copied,
0.5,
&[]
));
}
}
5 changes: 4 additions & 1 deletion crates/prism-review/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ mod prompts;
mod sim;
mod types;

pub use generic_arch::{coerce_generic_similarity, evidence_is_only_generic_tropes};
pub use generic_arch::{
cheap_similarity_hard_zeros, coerce_generic_similarity, evidence_is_only_generic_tropes,
SUSPICIOUS_HARD_ZERO_THRESHOLD,
};
pub use llm::{load_api_key_file, OpenRouterClient};
pub use prompts::{REVIEW_PROMPT_VERSION, SIMILARITY_PROMPT_VERSION};
pub use sim::SimReviewer;
Expand Down
2 changes: 1 addition & 1 deletion docs/COMPLETENESS.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`]
| prism Lium backend | done | `PRISM_FORCE_SIM=false` in staging; the binary logs `eval_backend=lium`. API key is mounted from a file so it never appears in `docker inspect`. |
| prism orchestration | done | DB-backed claim/execute/review/similarity/score state machine (`prism_submission` + append-only `prism_stage_event`), pre-pod screens (copy gate + static cheat + AST similarity) before Lium rent, sweeper (10h grace + pre-reclaim log harvest), boot recovery, epoch-close batched D24 leaf emission with **WTA** (`prism-emit` outbox: `emitted_epoch` watermark + `prism_emit_cursor` + positive-score carry + `apply_wta`, migration 0012). `PRISM_MAX_CONCURRENT_EVALS` default/prod = 8. |
| prism recipe v1 | done | `prism-recipe` contract, fineweb-edu pinned shard (URL + SHA-256, harness re-verifies), 6h train / 7h pod caps, baseline sources, recipe pin hex on the API. |
| prism LLM review | done | `prism-review` quality + similarity-v3 prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; cheap `Copied` hard-zeros, cheap `Suspicious` advisory; generic LM tropes coerced; copy/similarity/agentic corpus = champions (Score>0) + baseline. |
| prism LLM review | done | `prism-review` quality + similarity-v3 prompts (versioned), OpenRouter client (key file only, never env), deterministic sim fallback; cheap `Copied` hard-zeros; cheap `Suspicious` hard-zeros only at `score ≥ 0.9` with non-trope evidence (`SUSPICIOUS_HARD_ZERO_THRESHOLD`); generic LM tropes coerced; copy/similarity/agentic corpus = champions (Score>0) + baseline. |
| prism API | done | Full status surface: submissions list/detail/events/status/jobs/recipe/baseline, idempotent accept. |
| Phala / agent-v1 miner path | removed | External miners use HTTP submit only ([`external-miner/`](external-miner/)). |

Expand Down
28 changes: 17 additions & 11 deletions docs/PRISM.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ OpenRouter when keyed, `SimAgent` in CI). The LLM review also enforces the
**telemetry contract**: `training.py` must call `prism_telemetry.report(...)`
+ `prism_telemetry.finish_evaluation()`; missing hooks are a hard contract
violation (`missing_telemetry_hooks` → `Score(0)`, terminal). Cheap
`Copied` is a hard first filter; cheap `Suspicious` is advisory. Agentic is
the primary anti-cheat judge and must not treat standard LM components as
plagiarism. The LLM quality vote is a **coherence gate, never a grader**:
the final score is pure bpb, with hard-zero on agentic `cheat`/`suspicious`
and cheap `Copied`. Missing agentic verdict is fail-closed
(`ChallengeInternal`). Leaves are D24-complete per chain epoch, emitted at
`Copied` is a hard first filter; cheap `Suspicious` hard-zeros only when
`score ≥ 0.9` (`SUSPICIOUS_HARD_ZERO_THRESHOLD`) and evidence is not
generic-trope-only. Agentic is the primary anti-cheat judge and must not
treat standard LM components as plagiarism. The LLM quality vote is a
**coherence gate, never a grader**: the final score is pure bpb, with
hard-zero on agentic `cheat`/`suspicious` and cheap `Copied` / high-confidence
`Suspicious`. Missing agentic verdict is fail-closed (`ChallengeInternal`).
Leaves are D24-complete per chain epoch, emitted at
epoch close from the finalized-since-last-epoch batch (see **Leaf emission**
below). Review findings are audit events, not points.

Expand Down Expand Up @@ -193,9 +195,11 @@ in order and terminal-reject with `Score(0)` on hit:
hardcoded `METRICS_JSON=` short-circuit; missing
`prism_telemetry.report` / `finish_evaluation` hooks in `training.py`.
3. **Cheap LLM similarity** (`prism-review` similarity-v3) — hard-zero on
`Copied` only before rent. `Suspicious` is advisory (does not reject).
Parsers coerce verdicts whose evidence is only standard LM components
(RMSNorm / RoPE / SwiGLU / LayerNorm / gated or parallel residual, …).
`Copied`, and on `Suspicious` when `score ≥ 0.9` with non-trope evidence
(`combine_final` + pre-pod share [`cheap_similarity_hard_zeros`]).
Below-threshold `Suspicious` (e.g. 0.7) does not wipe. Parsers coerce
verdicts whose evidence is only standard LM components (RMSNorm / RoPE /
SwiGLU / LayerNorm / gated or parallel residual, …).

After measure, the LLM quality review and the shared `challenge-agentic` loop
inspect sources + metrics/receipt with read-only tools (`list_dir`,
Expand All @@ -210,7 +214,7 @@ modern-LM components as plagiarism; AST bands (`≥8500` suspicious /
| `clean` | proceed; score = pure bpb on `[0, SCORE_MAX]` |
| agentic `suspicious` / `cheat` | `Score(0)` via `combine_final` |
| cheap LLM `Copied` | `Score(0)` |
| cheap LLM `Suspicious` | advisory only (not a hard zero) |
| cheap LLM `Suspicious` | `Score(0)` iff `score ≥ 0.9` and evidence not trope-only; else no wipe |
| missing / unparseable | `NoScore(ChallengeInternal)` (fail-closed) |

Cheat taxonomy (Prism-relevant):
Expand All @@ -224,7 +228,9 @@ Cheat taxonomy (Prism-relevant):
| `missing_telemetry_hooks` | `training.py` does not call `prism_telemetry.report` + `finish_evaluation` |

Cheap `Copied` from single-shot similarity remains a hard-zero first filter;
cheap `Suspicious` is advisory. Agentic is the **primary** anti-cheat judge.
cheap `Suspicious` uses the numeric score against
`SUSPICIOUS_HARD_ZERO_THRESHOLD` (0.9) plus trope coercion. Agentic is the
**primary** anti-cheat judge.
Public site gallery/leaderboard list **champions only** (Score>0); operators
still see the full corpus via the challenge API. LLM quality stays audit-only
for the bpb score (coherence gate, never a grader).
Expand Down
4 changes: 3 additions & 1 deletion docs/PRISM_RECIPE.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,9 @@ verdict, quality notes and issues are kept as audit records
review still gates eligibility:

- similarity verdict `Copied` → hard **Score 0**
- similarity verdict `Suspicious` → advisory only (not a hard zero; agentic is the judge)
- similarity verdict `Suspicious` → **Score 0** only when `score ≥ 0.9` and
evidence is not generic-trope-only (else no wipe; agentic remains the
structural judge)
- harness/antipattern failure → `ChallengeInternal` maps to `NoScore` reason

## Anti-copy review
Expand Down
14 changes: 8 additions & 6 deletions docs/external-miner/prism.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,14 @@ submission and never rents a Lium pod.

Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`.
The shared **agentic** gate (AST + metrics/receipt) hard-zeros `cheat` /
`suspicious`. Cheap LLM similarity hard-zeros only `Copied` (`Suspicious` is
advisory). Copy/similarity corpora are **champions only** (current top +
historical Score>0 ex-tops) plus baseline — not every past submission — and
still exclude your own prior art (same hotkey **or** same coldkey). Standard
components (RMSNorm, RoPE, SwiGLU, LayerNorm, gated/parallel residual, …) are
**not** plagiarism signals. LLM quality is coherence-only, not a grader.
`suspicious`. Cheap LLM similarity hard-zeros `Copied`, and `Suspicious` only
when confidence `≥ 0.9` with non-generic evidence (below that — e.g. 0.7 citing
RMSNorm/SwiGLU/LayerNorm — does **not** wipe your score). Copy/similarity
corpora are **champions only** (current top + historical Score>0 ex-tops) plus
baseline — not every past submission — and still exclude your own prior art
(same hotkey **or** same coldkey). Standard components (RMSNorm, RoPE, SwiGLU,
LayerNorm, gated/parallel residual, …) are **not** plagiarism signals. LLM
quality is coherence-only, not a grader.
Public gallery/leaderboard show champions only.
**Competition:** per epoch you are
credited the max of (a) your own best training result and (b) for each arch you
Expand Down
2 changes: 1 addition & 1 deletion docs/external-miner/troubleshoot.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
| Symptom | Likely cause | What to check |
|---------|--------------|---------------|
| Rejected submit | Recipe contract | `GET /v1/recipe` + baseline; follow [`PRISM_RECIPE.md`](../PRISM_RECIPE.md) |
| Score 0 after review | `Copied` / `Suspicious` | Similarity gate; rewrite; do not paste baseline wholesale |
| Score 0 after review | `Copied` / high-confidence `Suspicious` (≥0.9, non-trope) | Similarity gate; rewrite unique structure; tropes alone are not plagiarism |
| `similar: true` on precheck | Would hit intake copy gate | Rewrite `architecture.py`; baseline is fine to start from |
| `429 precheck_quota_exceeded` | 3 prechecks/coldkey/UTC day used | Wait until next UTC day; rotating hotkeys does not reset |
| Stuck `Provisioning` | Lium market thinness | Ops-side; watch `GET /v1/jobs` / events |
Expand Down
Loading