Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bins/prism-challenge/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ struct Cli {
#[arg(
long,
env = "PRISM_MAX_CONCURRENT_EVALS",
default_value_t = 1,
default_value_t = 8,
global = true
)]
max_concurrent_evals: u32,
Expand Down
5 changes: 4 additions & 1 deletion crates/challenge-agentic/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,10 @@ mod tools;
mod types;

pub use agent::{AgentConfig, OpenRouterAgent};
pub use challenge_ast::{copy_gate, CopyGateHit, GateCorpusEntry};
pub use challenge_ast::{
copy_gate, static_source_cheat, training_has_telemetry_hooks, CopyGateHit, GateCorpusEntry,
SourceCheatHit, SourceCheatKind,
};
pub use llm::{load_api_key_file, DEFAULT_MODEL};
pub use prompts::{AGENTIC_PROMPT_VERSION, DESIGN_DOMAIN_RULES, PRISM_DOMAIN_RULES};
pub use sim::{SimAgent, SIM_CHEAT_BPS, SIM_SUSPICIOUS_BPS};
Expand Down
7 changes: 1 addition & 6 deletions crates/challenge-agentic/src/sim.rs
Original file line number Diff line number Diff line change
Expand Up @@ -187,12 +187,7 @@ fn telemetry_hooks_verdict(
return None;
}
let (path, src) = primaries.iter().find(|(p, _)| p.ends_with("training.py"))?;
let imports_shim = src.contains("prism_telemetry")
|| src.contains("ctx[\"telemetry\"]")
|| src.contains("ctx['telemetry']");
let calls_report = src.contains(".report(");
let calls_finish = src.contains("finish_evaluation(");
if imports_shim && calls_report && calls_finish {
if challenge_ast::training_has_telemetry_hooks(src) {
return None;
}
Some(AgenticVerdict {
Expand Down
4 changes: 4 additions & 0 deletions crates/challenge-ast/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
mod fingerprint;
mod gate;
mod similarity;
mod source_cheats;
mod walk;

pub use fingerprint::{fingerprint_source, AstError, Fingerprint};
Expand All @@ -19,6 +20,9 @@ pub use gate::{
pub use similarity::{
similarity_bps, structural_diff_summary, summarize_fingerprint, top_k_nearest, Neighbor,
};
pub use source_cheats::{
static_source_cheat, training_has_telemetry_hooks, SourceCheatHit, SourceCheatKind,
};

/// Crate identity smoke.
#[must_use]
Expand Down
89 changes: 89 additions & 0 deletions crates/challenge-ast/src/source_cheats.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
//! Cheap source-only cheat screens (no GPU, no private eval assets).

/// Kind of static source hit (maps to agentic `CheatCode` at the call site).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SourceCheatKind {
/// Hardcoded `METRICS_JSON=` short-circuit.
EvalShortCircuit,
/// Missing Prism telemetry hooks in `training.py`.
MissingTelemetryHooks,
}

/// One static source finding.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceCheatHit {
/// Cheat kind.
pub kind: SourceCheatKind,
/// Human-readable reason (safe to surface in `error_detail`).
pub rationale: String,
}

/// Scan miner sources for cheap, deterministic cheat patterns.
#[must_use]
pub fn static_source_cheat(architecture_py: &str, training_py: &str) -> Option<SourceCheatHit> {
for (path, src) in [
("architecture.py", architecture_py),
("training.py", training_py),
] {
if src.contains("METRICS_JSON=") {
return Some(SourceCheatHit {
kind: SourceCheatKind::EvalShortCircuit,
rationale: format!("static: hardcoded METRICS_JSON in {path}"),
});
}
}
if !training_has_telemetry_hooks(training_py) {
return Some(SourceCheatHit {
kind: SourceCheatKind::MissingTelemetryHooks,
rationale: "static: training.py missing prism_telemetry report/finish_evaluation hooks"
.into(),
});
}
None
}

/// Prism telemetry-hook contract (recipe ≥ 1.1.0).
#[must_use]
pub fn training_has_telemetry_hooks(training_py: &str) -> bool {
let imports_shim = training_py.contains("prism_telemetry")
|| training_py.contains("ctx[\"telemetry\"]")
|| training_py.contains("ctx['telemetry']");
training_py.contains(".report(") && training_py.contains("finish_evaluation(") && imports_shim
}

#[cfg(test)]
mod tests {
use super::*;

#[test]
fn metrics_json_short_circuit() {
let hit = static_source_cheat(
"def build_model(ctx):\n pass\n",
"def train(m, ctx):\n print('METRICS_JSON={}')\n",
)
.expect("hit");
assert_eq!(hit.kind, SourceCheatKind::EvalShortCircuit);
}

#[test]
fn missing_hooks() {
let hit = static_source_cheat(
"def build_model(ctx):\n pass\n",
"def train(m, ctx):\n return {}\n",
)
.expect("hit");
assert_eq!(hit.kind, SourceCheatKind::MissingTelemetryHooks);
}

#[test]
fn clean_hooks() {
let train = concat!(
"import prism_telemetry\n",
"def train(m, ctx):\n",
" prism_telemetry.report(loss=1.0, step=1)\n",
" prism_telemetry.finish_evaluation()\n",
" return {}\n",
);
assert!(static_source_cheat("def build_model(ctx):\n pass\n", train).is_none());
}
}
3 changes: 2 additions & 1 deletion crates/prism-challenge/src/leaf_emit.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use challenge_common::{emit_signed_leaf_set as emit_signed_leaf_set_common, Hotk
use crypto::KEY_LEN;
use prism_challenge_task::CHALLENGE_ID_BYTES;

pub use challenge_common::{public_key_from_secret, verify_leaf_sig, LeafEmitError};
use challenge_common::LeafEmitError;

/// Sign exactly one leaf per `h ∈ expected` under `prism`. Refuses subset/superset (D24).
///
Expand All @@ -27,6 +27,7 @@ mod tests {
#![allow(clippy::unwrap_used)]
use super::*;
use bundle::ScoreOrAbsence;
use challenge_common::{public_key_from_secret, verify_leaf_sig};
use crypto::KEY_LEN;

fn sk() -> [u8; KEY_LEN] {
Expand Down
10 changes: 5 additions & 5 deletions crates/prism-challenge/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,13 @@ mod api;
mod leaf_emit;
pub mod orchestrator;
mod score;
mod submit;

pub use api::{record_epoch, submission_router, AppState};
pub use leaf_emit::{emit_signed_leaf_set, public_key_from_secret, verify_leaf_sig, LeafEmitError};
pub use challenge_common::{
public_key_from_secret, submit_signed_leaf_set, verify_leaf_sig, GatewayClient,
GatewayClientConfig, LeafEmitError, SubmitError, SubmitOutcome,
};
pub use leaf_emit::emit_signed_leaf_set;
pub use orchestrator::{Orchestrator, OrchestratorConfig};
pub use prism_challenge_task::{
CHALLENGE_ID, CHALLENGE_ID_BYTES, SCORE_MAX, SCORING_VERSION, TASK_ID_DOMAIN,
Expand All @@ -36,9 +39,6 @@ pub use prism_store::{
StoreError, SubmissionState,
};
pub use score::{combine_final, FinalOutcome};
pub use submit::{
submit_signed_leaf_set, GatewayClient, GatewayClientConfig, SubmitError, SubmitOutcome,
};

pub use bundle::{LeafV1, NoScoreReasonCode, ScoreOrAbsence};
pub use crypto::KEY_LEN;
Expand Down
Loading
Loading