From c65156b12bf43e38943f35a8c281fab37a20f43a Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 06:49:21 +0000 Subject: [PATCH] feat(prism): add similarity precheck API with coldkey daily quota Let miners dry-run the intake copy gate before burning a 1-max slot or Lium pod. Cap at 3 checks per coldkey per UTC day so hotkey rotation cannot spam. --- Cargo.lock | 2 + .../migrations/0016_prism_precheck_quota.sql | 17 + crates/db/src/prism_store.rs | 47 +++ crates/prism-challenge/src/agentic.rs | 166 +------- crates/prism-challenge/src/api.rs | 240 ++++++++++-- crates/prism-pipeline/Cargo.toml | 2 + crates/prism-pipeline/src/lib.rs | 7 + crates/prism-pipeline/src/precheck.rs | 353 ++++++++++++++++++ crates/prism-store/src/dbprism.rs | 20 + crates/prism-store/src/store.rs | 44 +++ docs/PRISM.md | 23 ++ docs/external-miner/prism.md | 27 ++ docs/external-miner/troubleshoot.md | 2 + 13 files changed, 761 insertions(+), 189 deletions(-) create mode 100644 crates/db/migrations/0016_prism_precheck_quota.sql create mode 100644 crates/prism-pipeline/src/precheck.rs diff --git a/Cargo.lock b/Cargo.lock index 177c1b050..e4e2edff5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3837,10 +3837,12 @@ version = "0.1.0" dependencies = [ "base64 0.22.1", "bundle", + "challenge-agentic", "hex", "prism-challenge-task", "prism-lium", "prism-recipe", + "prism-store", "serde", "serde_json", "sha2 0.10.9", diff --git a/crates/db/migrations/0016_prism_precheck_quota.sql b/crates/db/migrations/0016_prism_precheck_quota.sql new file mode 100644 index 000000000..f1d8ff165 --- /dev/null +++ b/crates/db/migrations/0016_prism_precheck_quota.sql @@ -0,0 +1,17 @@ +-- Prism miner similarity precheck quota: 3 attempts per coldkey per UTC day. +-- +-- `POST /v1/submissions/precheck` runs the same pre-LLM copy gate as intake +-- without creating a submission or renting a pod. Quota is keyed by coldkey +-- (hotkey fallback when Owner is unknown) so rotating hotkeys cannot reset +-- the daily budget. + +CREATE TABLE prism_precheck_quota ( + miner_coldkey TEXT NOT NULL, + day DATE NOT NULL, + checks_used INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (miner_coldkey, day), + CONSTRAINT prism_precheck_quota_key_hex CHECK (miner_coldkey ~ '^[0-9a-f]{64}$'), + CONSTRAINT prism_precheck_quota_checks_nonneg CHECK (checks_used >= 0) +); + +GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE prism_precheck_quota TO base_app; diff --git a/crates/db/src/prism_store.rs b/crates/db/src/prism_store.rs index 0d4799436..b2fc6cead 100644 --- a/crates/db/src/prism_store.rs +++ b/crates/db/src/prism_store.rs @@ -322,3 +322,50 @@ pub async fn stuck_prism_before_grace( .await?; Ok(rows) } + +/// Read precheck attempts used for `(coldkey, UTC day)` (0 when absent). +/// +/// # Errors +/// SQL error. +pub async fn prism_precheck_quota_get( + pool: &PgPool, + miner_coldkey: &str, + day: &str, +) -> Result { + let row: Option<(i32,)> = sqlx::query_as( + "SELECT checks_used FROM prism_precheck_quota \ + WHERE miner_coldkey = $1 AND day = $2::date", + ) + .bind(miner_coldkey) + .bind(day) + .fetch_optional(pool) + .await?; + Ok(row.map_or(0, |r| r.0)) +} + +/// Atomically consume one precheck attempt when `checks_used < limit`. +/// Returns `Some(checks_used)` after bump, or `None` when already at limit. +/// +/// # Errors +/// SQL error. +pub async fn prism_precheck_quota_try_consume( + pool: &PgPool, + miner_coldkey: &str, + day: &str, + limit: i32, +) -> Result, DbError> { + let row: Option<(i32,)> = sqlx::query_as( + "INSERT INTO prism_precheck_quota (miner_coldkey, day, checks_used) \ + VALUES ($1, $2::date, 1) \ + ON CONFLICT (miner_coldkey, day) DO UPDATE SET \ + checks_used = prism_precheck_quota.checks_used + 1 \ + WHERE prism_precheck_quota.checks_used < $3 \ + RETURNING checks_used", + ) + .bind(miner_coldkey) + .bind(day) + .bind(limit) + .fetch_optional(pool) + .await?; + Ok(row.map(|r| r.0)) +} diff --git a/crates/prism-challenge/src/agentic.rs b/crates/prism-challenge/src/agentic.rs index ca9d6e74f..830985fd4 100644 --- a/crates/prism-challenge/src/agentic.rs +++ b/crates/prism-challenge/src/agentic.rs @@ -1,25 +1,16 @@ -//! Workdir + corpus helpers for the Prism agentic anti-cheat gate. +//! Workdir helpers for the Prism agentic anti-cheat gate. +//! +//! Corpus builders live in [`prism_pipeline::precheck`] (LOC split); this +//! module only materializes the review workdir. use std::fs; use std::path::Path; -use challenge_agentic::{ - same_miner_identity, CorpusEntry, GateCorpusEntry, ReviewRequest, PRISM_DOMAIN_RULES, -}; +use challenge_agentic::{CorpusEntry, ReviewRequest, PRISM_DOMAIN_RULES}; use prism_lium::{EvalReceipt, RemoteExecResult}; -use prism_recipe::BASELINE_ARCHITECTURE_PY; use prism_store::SubmissionState; -/// True when `other` is the same economic miner as `candidate` (hotkey or coldkey). -#[must_use] -pub fn same_miner(candidate: &SubmissionState, other: &SubmissionState) -> bool { - same_miner_identity( - &candidate.miner_hotkey, - candidate.miner_coldkey.as_deref(), - &other.miner_hotkey, - other.miner_coldkey.as_deref(), - ) -} +pub use prism_pipeline::{corpus_from_rows, gate_corpus_from_rows, same_miner}; /// Build a temp workdir + [`ReviewRequest`] for one Prism submission. /// @@ -59,148 +50,3 @@ pub fn build_review_request( domain_rules: PRISM_DOMAIN_RULES.into(), }) } - -/// Pre-LLM copy-gate corpus: other miners' prior art only (hotkey + coldkey). -#[must_use] -pub fn gate_corpus_from_rows( - candidate: &SubmissionState, - recent: &[SubmissionState], -) -> Vec { - recent - .iter() - .filter(|r| r.id != candidate.id && !same_miner(candidate, r)) - .map(|r| GateCorpusEntry { - id: format!("subm:{}", r.id), - source: r.architecture_py.clone(), - created_at_ms: r.created_at_ms, - }) - .collect() -} - -/// Baseline + recent terminated submissions as agentic corpus entries. -/// -/// Corpus entries are **architecture.py only** (similarity v2): `training.py` -/// is exempt from every copy/similarity comparison — the same training -/// script on two different architectures is legitimate competition behavior. -/// `exempt_arch` drops entries byte-equal to that source (training-only -/// submissions on a registry architecture: the identity is by design). -/// Same-hotkey and same-coldkey prior art are excluded. -#[must_use] -pub fn corpus_from_rows( - candidate: &SubmissionState, - recent: &[SubmissionState], - exempt_arch: Option<&str>, -) -> Vec { - let mut v = vec![CorpusEntry { - id: "baseline".into(), - source: BASELINE_ARCHITECTURE_PY.into(), - }]; - for r in recent { - if r.id == candidate.id || same_miner(candidate, r) { - continue; - } - if Some(r.architecture_py.as_str()) == exempt_arch { - continue; - } - let label = if r.id.len() >= 8 { - format!("subm:{}", &r.id[..8]) - } else { - format!("subm:{}", r.id) - }; - v.push(CorpusEntry { - id: label, - source: r.architecture_py.clone(), - }); - } - v -} - -#[cfg(test)] -mod tests { - use super::*; - use prism_store::{FinalScore, Stage}; - - fn row( - id: &str, - hotkey: &str, - coldkey: Option<&str>, - arch: &str, - created_at_ms: u64, - ) -> SubmissionState { - SubmissionState { - id: id.into(), - miner_hotkey: hotkey.into(), - miner_coldkey: coldkey.map(str::to_owned), - epoch: 1, - netuid: 1, - status: Stage::Terminated, - architecture_py: arch.into(), - training_py: "train".into(), - label: None, - pod_id: None, - pod_provider: None, - receipt: None, - metrics_json: None, - bpb: Some(1.0), - arch_id: None, - review: None, - similarity: None, - final_score: Some(FinalScore::Score(1)), - retry_count: 0, - error_detail: None, - created_at_ms, - updated_at_ms: created_at_ms, - } - } - - #[test] - fn same_hotkey_prior_art_excluded() { - let prior = row("aaaaaaaa", "aa", Some("11"), "arch_a", 1_000); - let next = row("bbbbbbbb", "aa", Some("11"), "arch_b", 2_000); - let recent = vec![prior, next.clone()]; - assert!(gate_corpus_from_rows(&next, &recent).is_empty()); - assert_eq!( - corpus_from_rows(&next, &recent, None) - .iter() - .map(|e| e.id.as_str()) - .collect::>(), - vec!["baseline"] - ); - } - - #[test] - fn same_coldkey_different_hotkey_excluded() { - let prior = row("aaaaaaaa", "aa", Some("11"), "arch_a", 1_000); - let next = row("bbbbbbbb", "bb", Some("11"), "arch_b", 2_000); - let recent = vec![prior, next.clone()]; - assert!(gate_corpus_from_rows(&next, &recent).is_empty()); - assert_eq!( - corpus_from_rows(&next, &recent, None) - .iter() - .map(|e| e.id.as_str()) - .collect::>(), - vec!["baseline"] - ); - } - - #[test] - fn different_coldkey_stays_in_corpus() { - let victim = row("aaaaaaaa", "aa", Some("11"), "arch_a", 1_000); - let copier = row("bbbbbbbb", "bb", Some("22"), "arch_b", 2_000); - let recent = vec![victim, copier.clone()]; - assert_eq!( - gate_corpus_from_rows(&copier, &recent) - .iter() - .map(|e| e.id.as_str()) - .collect::>(), - vec!["subm:aaaaaaaa"] - ); - assert_eq!( - corpus_from_rows(&copier, &recent, None) - .iter() - .map(|e| e.id.as_str()) - .collect::>(), - vec!["baseline", "subm:aaaaaaaa"] - ); - } -} diff --git a/crates/prism-challenge/src/api.rs b/crates/prism-challenge/src/api.rs index 04a9f09ce..4f3f88704 100644 --- a/crates/prism-challenge/src/api.rs +++ b/crates/prism-challenge/src/api.rs @@ -4,6 +4,7 @@ //! |-------|---------| //! | `GET /health` | liveness | //! | `POST /v1/submissions` | accept a two-script recipe | +//! | `POST /v1/submissions/precheck` | advisory copy-gate (quota 3/coldkey/UTC day) | //! | `GET /v1/submissions` | list (`status` / `miner` filter, limit) | //! | `GET /v1/submissions/{id}` | full detail + event timeline | //! | `GET /v1/submissions/{id}/events` | journal only | @@ -30,7 +31,11 @@ use submission_gating::{GatingState, GatingStore, MetagraphCache}; use prism_recipe::{BASELINE_ARCHITECTURE_PY, BASELINE_TRAINING_PY}; use crate::CHALLENGE_ID; -use prism_pipeline::{SubmissionError, SubmissionRequest}; +use prism_pipeline::{ + ephemeral_candidate, evaluate_copy_precheck, precheck_json, precheck_quota_exceeded_json, + precheck_skipped, quota_identity, quota_view, utc_day, SubmissionError, SubmissionRequest, + PRECHECK_DAILY_LIMIT, +}; use prism_store::{FinalScore, PrismStore, Stage, StoreError, SubmissionState}; /// Shared HTTP app state. @@ -58,6 +63,7 @@ pub fn submission_router(state: Arc) -> Router { Router::new() .route("/health", get(health)) .route("/v1/submissions", post(post_submission)) + .route("/v1/submissions/precheck", post(post_precheck)) .route("/v1/submissions", get(list_submissions)) .route("/v1/submissions/{id}", get(get_submission)) .route("/v1/submissions/{id}/events", get(get_events)) @@ -114,39 +120,37 @@ fn parse_submission_body( serde_json::from_slice(body).map_err(|e| format!("invalid_json: {e}")) } -/// Intake gates for a fresh submission: metagraph membership (fail closed -/// when the cache has no snapshot) + one accepted submission per -/// `(challenge, hotkey)`. `challenge` is `prism` for architecture -/// submissions (1-max per hotkey) or `prism:train:` for -/// training-only entries (1 accepted entry per `(hotkey, arch_id)`). -/// Returns the metagraph uid on pass. +/// Metagraph membership only (fail closed when configured but empty). +#[allow(clippy::result_large_err)] // mirrors other intake helpers returning `Response` +fn metagraph_uid(st: &AppState, hotkey: &str) -> Result, Response> { + let Some(cache) = &st.metagraph else { + return Ok(None); + }; + match cache.snapshot() { + Some(view) => match view.uid_of_hex(hotkey) { + Some(u) => Ok(Some(u)), + None => Err(json_err( + StatusCode::FORBIDDEN, + "hotkey_not_in_metagraph", + "miner hotkey is not registered on this subnet", + )), + }, + None => Err(json_err( + StatusCode::SERVICE_UNAVAILABLE, + "metagraph_unavailable", + "metagraph snapshot not ready; retry shortly", + )), + } +} + +/// Intake gates: metagraph membership + one accepted submission per +/// `(challenge, hotkey)`. Returns the metagraph uid on pass. async fn intake_gates( st: &AppState, hotkey: &str, challenge: &str, ) -> Result, Response> { - let mut uid = None; - if let Some(cache) = &st.metagraph { - match cache.snapshot() { - Some(view) => match view.uid_of_hex(hotkey) { - Some(u) => uid = Some(u), - None => { - return Err(json_err( - StatusCode::FORBIDDEN, - "hotkey_not_in_metagraph", - "miner hotkey is not registered on this subnet", - )); - } - }, - None => { - return Err(json_err( - StatusCode::SERVICE_UNAVAILABLE, - "metagraph_unavailable", - "metagraph snapshot not ready; retry shortly", - )); - } - } - } + let uid = metagraph_uid(st, hotkey)?; gate_one_max(st, hotkey, challenge).await?; Ok(uid) } @@ -213,6 +217,78 @@ async fn materialize_arch(st: &AppState, req: &mut SubmissionRequest) -> Result< } } +/// `POST /v1/submissions/precheck` — advisory copy-gate (same logic as +/// intake), no submission row, no 1-max gate, no Lium. Quota: 3/coldkey/UTC day. +async fn post_precheck( + State(st): State>, + headers: axum::http::HeaderMap, + body: bytes::Bytes, +) -> Response { + let mut req = match parse_submission_body(&headers, body.as_ref()) { + Ok(r) => r, + Err(e) => return json_err(StatusCode::BAD_REQUEST, "invalid_submission", &e), + }; + if let Err(e) = prism_pipeline::expand_zip_fields(&mut req) { + return json_err(StatusCode::BAD_REQUEST, "zip", &e); + } + if let Err(e) = prism_pipeline::validate(&req) { + return map_submission_err(&e); + } + if let Err(resp) = materialize_arch(&st, &mut req).await { + return resp; + } + if let Err(e) = prism_recipe::check_contract(&req.architecture_py, &req.training_py) { + return json_err(StatusCode::BAD_REQUEST, "contract", &e.to_string()); + } + req.miner_hotkey = req.miner_hotkey.trim().to_ascii_lowercase(); + if let Err(resp) = metagraph_uid(&st, &req.miner_hotkey) { + return resp; + } + let miner_coldkey = st + .metagraph + .as_ref() + .and_then(|c| c.snapshot()) + .and_then(|v| v.coldkey_hex_of(&req.miner_hotkey)); + let (identity, identity_kind) = quota_identity(&req.miner_hotkey, miner_coldkey.as_deref()); + let day = utc_day(now_ms() / 1000); + let used = match st + .store + .precheck_quota_try_consume(&identity, &day, PRECHECK_DAILY_LIMIT) + .await + { + Ok(Some(n)) => n, + Ok(None) => { + let used = st + .store + .precheck_quota_get(&identity, &day) + .await + .unwrap_or(PRECHECK_DAILY_LIMIT); + let q = quota_view(day, used, identity_kind); + return ( + StatusCode::TOO_MANY_REQUESTS, + Json(precheck_quota_exceeded_json(&q)), + ) + .into_response(); + } + Err(e) => { + return json_err(StatusCode::INTERNAL_SERVER_ERROR, "store", &e.to_string()); + } + }; + let quota = quota_view(day, used, identity_kind); + if req.arch_id.is_some() { + return Json(precheck_json(&precheck_skipped(quota))).into_response(); + } + let candidate = ephemeral_candidate( + &req.miner_hotkey, + miner_coldkey, + &req.architecture_py, + now_ms(), + ); + let recent = st.store.list(None, None, 64).await.unwrap_or_default(); + let result = evaluate_copy_precheck(&candidate, &recent, quota); + Json(precheck_json(&result)).into_response() +} + /// POST body: JSON sources, JSON+`zip_base64`, or raw `application/zip` /// with `X-Miner-Hotkey`. async fn post_submission( @@ -1062,6 +1138,112 @@ mod tests { assert_eq!(v["status"], "already-queued"); } + #[tokio::test] + async fn precheck_detects_copy_without_queuing() { + let st = state(); + let victim_hk = "aa".repeat(32); + let mut victim = crate::example_valid_request(); + victim.miner_hotkey = victim_hk; + let vid = prism_pipeline::submission_id(&victim); + let mut row = SubmissionState { + id: vid, + miner_hotkey: victim.miner_hotkey.clone(), + miner_coldkey: Some("11".repeat(32)), + epoch: 1, + netuid: 541, + status: Stage::Terminated, + architecture_py: victim.architecture_py.clone(), + training_py: victim.training_py.clone(), + label: None, + pod_id: None, + pod_provider: None, + receipt: None, + metrics_json: None, + bpb: Some(1.0), + arch_id: None, + review: None, + similarity: None, + final_score: Some(FinalScore::Score(1)), + retry_count: 0, + error_detail: None, + created_at_ms: 1_000, + updated_at_ms: 1_000, + }; + // Ensure created_at is in the past relative to precheck `now_ms`. + row.created_at_ms = 1; + st.store.insert_queued(&row).await.unwrap(); + // Force terminated without going through claim (memory insert is queued). + st.store + .apply( + &row.id, + &StatePatch { + status: Some(Stage::Terminated), + final_score: Some(FinalScore::Score(1)), + ..StatePatch::default() + }, + None, + ) + .await + .unwrap(); + + let app = submission_router(Arc::clone(&st)); + let mut copy = crate::example_valid_request(); + copy.miner_hotkey = "bb".repeat(32); + copy.architecture_py = victim.architecture_py; + let body = serde_json::to_vec(©).unwrap(); + let (s, v) = call( + app.clone(), + Request::post("/v1/submissions/precheck") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::OK, "{v}"); + assert_eq!(v["similar"], true); + assert_eq!(v["verdict"], "copied"); + assert!(v["matched_against"].as_str().unwrap().starts_with("subm:")); + assert_eq!(v["quota"]["used"], 1); + assert_eq!(v["quota"]["remaining"], 2); + // No new submission row for the copier. + let listed = st.store.list(None, None, 50).await.unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].miner_hotkey, "aa".repeat(32)); + } + + #[tokio::test] + async fn precheck_quota_is_three_then_429() { + let st = state(); + let app = submission_router(Arc::clone(&st)); + let body = serde_json::to_vec(&crate::example_valid_request()).unwrap(); + for i in 1..=3 { + let (s, v) = call( + app.clone(), + Request::post("/v1/submissions/precheck") + .header("content-type", "application/json") + .body(Body::from(body.clone())) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::OK, "attempt {i}: {v}"); + assert_eq!(v["similar"], false); + assert_eq!(v["verdict"], "clean"); + assert_eq!(v["quota"]["used"], i); + } + let (s, v) = call( + app, + Request::post("/v1/submissions/precheck") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await; + assert_eq!(s, StatusCode::TOO_MANY_REQUESTS, "{v}"); + assert_eq!(v["code"], "precheck_quota_exceeded"); + assert_eq!(v["quota"]["remaining"], 0); + assert_eq!(v["quota"]["used"], 3); + } + #[tokio::test] async fn gated_intake_503_until_first_snapshot() { // Cache configured but never refreshed → fail closed with 503. diff --git a/crates/prism-pipeline/Cargo.toml b/crates/prism-pipeline/Cargo.toml index 573f39653..9f4e14548 100644 --- a/crates/prism-pipeline/Cargo.toml +++ b/crates/prism-pipeline/Cargo.toml @@ -11,10 +11,12 @@ publish = false [dependencies] base64 = "0.22" bundle = { path = "../bundle" } +challenge-agentic = { path = "../challenge-agentic" } hex = "0.4" prism-challenge-task = { path = "../prism-challenge-task" } prism-lium = { path = "../prism-lium" } prism-recipe = { path = "../prism-recipe" } +prism-store = { path = "../prism-store" } serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" diff --git a/crates/prism-pipeline/src/lib.rs b/crates/prism-pipeline/src/lib.rs index fb84bf24a..510d32917 100644 --- a/crates/prism-pipeline/src/lib.rs +++ b/crates/prism-pipeline/src/lib.rs @@ -11,6 +11,7 @@ pub mod config; pub mod pipeline; +pub mod precheck; pub mod score; pub mod submission; @@ -18,6 +19,12 @@ pub use config::PrismConfig; pub use pipeline::{ run_eval_pipeline, run_sim_pipeline, PipelineError, PipelineInput, PipelineResult, }; +pub use precheck::{ + corpus_from_rows, ephemeral_candidate, evaluate_copy_precheck, gate_corpus_from_rows, + precheck_json, precheck_quota_exceeded_json, precheck_skipped, quota_identity, quota_view, + same_miner, utc_day, PrecheckIdentityKind, PrecheckQuotaView, PrecheckResult, + PRECHECK_DAILY_LIMIT, +}; pub use score::{score_from_bpb, score_from_pipeline, PipelineOutcome}; pub use submission::{ arch_digest, arch_id_for, example_valid_request, expand_zip_fields, gating_key, is_arch_id, diff --git a/crates/prism-pipeline/src/precheck.rs b/crates/prism-pipeline/src/precheck.rs new file mode 100644 index 000000000..948f4ca90 --- /dev/null +++ b/crates/prism-pipeline/src/precheck.rs @@ -0,0 +1,353 @@ +//! Miner-facing similarity precheck (copy gate only — no pod, no LLM). + +use challenge_agentic::{ + copy_gate, same_miner_identity, CopyGateHit, CorpusEntry, GateCorpusEntry, +}; +use prism_recipe::BASELINE_ARCHITECTURE_PY; +use prism_store::SubmissionState; +use serde::Serialize; +use serde_json::{json, Value}; + +/// Max precheck attempts per coldkey (or hotkey fallback) per UTC day. +pub const PRECHECK_DAILY_LIMIT: u32 = 3; + +/// Quota key axis reported to miners. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PrecheckIdentityKind { + /// Metagraph Owner coldkey. + Coldkey, + /// Hotkey used when coldkey is unknown (zeros / no snapshot). + HotkeyFallback, +} + +/// Daily precheck budget snapshot. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct PrecheckQuotaView { + /// UTC day `YYYY-MM-DD`. + pub day: String, + /// Attempts consumed (including the current one when allowed). + pub used: u32, + /// Hard cap ([`PRECHECK_DAILY_LIMIT`]). + pub limit: u32, + /// Attempts left today. + pub remaining: u32, + /// Whether the counter key is coldkey or hotkey fallback. + pub identity: PrecheckIdentityKind, +} + +/// Result of the advisory copy-gate precheck. +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct PrecheckResult { + /// True when the architecture would hard-reject at intake copy gate. + pub similar: bool, + /// `clean` | `copied` | `skipped`. + pub verdict: &'static str, + /// Corpus id of the nearest earlier hit (never full competitor source). + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_against: Option, + /// Similarity in `[0, 1]` (`bps / 10000`), when compared. + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, + /// Human-readable guidance (no competitor source). + pub message: String, + /// Present on copy hits. + #[serde(skip_serializing_if = "Option::is_none")] + pub byte_identical: Option, + /// Daily budget after this call (or at rejection). + pub quota: PrecheckQuotaView, +} + +/// UTC calendar day `YYYY-MM-DD` from unix seconds. +#[must_use] +pub fn utc_day(secs: u64) -> String { + let days = secs / 86_400; + let z = i64::try_from(days).unwrap_or(i64::MAX) + 719_468; + let era = if z >= 0 { z } else { z - 146_096 } / 146_097; + let doe = u64::try_from(z - era * 146_097).unwrap_or(0); + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365; + let y = i64::try_from(yoe).unwrap_or(0) + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{y:04}-{m:02}-{d:02}") +} + +/// Resolve the quota counter key: prefer coldkey, else hotkey (both 64 hex). +#[must_use] +pub fn quota_identity(hotkey: &str, coldkey: Option<&str>) -> (String, PrecheckIdentityKind) { + match coldkey.map(str::trim).filter(|s| !s.is_empty()) { + Some(ck) => (ck.to_ascii_lowercase(), PrecheckIdentityKind::Coldkey), + None => ( + hotkey.trim().to_ascii_lowercase(), + PrecheckIdentityKind::HotkeyFallback, + ), + } +} + +/// Build a quota view from used count. +#[must_use] +pub fn quota_view( + day: impl Into, + used: u32, + identity: PrecheckIdentityKind, +) -> PrecheckQuotaView { + let limit = PRECHECK_DAILY_LIMIT; + PrecheckQuotaView { + day: day.into(), + used, + limit, + remaining: limit.saturating_sub(used), + identity, + } +} + +/// True when `other` is the same economic miner as `candidate`. +#[must_use] +pub fn same_miner(candidate: &SubmissionState, other: &SubmissionState) -> bool { + same_miner_identity( + &candidate.miner_hotkey, + candidate.miner_coldkey.as_deref(), + &other.miner_hotkey, + other.miner_coldkey.as_deref(), + ) +} + +/// Pre-LLM copy-gate corpus: other miners' prior art only (hotkey + coldkey). +#[must_use] +pub fn gate_corpus_from_rows( + candidate: &SubmissionState, + recent: &[SubmissionState], +) -> Vec { + recent + .iter() + .filter(|r| r.id != candidate.id && !same_miner(candidate, r)) + .map(|r| GateCorpusEntry { + id: format!("subm:{}", r.id), + source: r.architecture_py.clone(), + created_at_ms: r.created_at_ms, + }) + .collect() +} + +/// Baseline + recent terminated submissions as agentic corpus entries. +/// +/// Architecture.py only; same-hotkey and same-coldkey prior art excluded. +#[must_use] +pub fn corpus_from_rows( + candidate: &SubmissionState, + recent: &[SubmissionState], + exempt_arch: Option<&str>, +) -> Vec { + let mut v = vec![CorpusEntry { + id: "baseline".into(), + source: BASELINE_ARCHITECTURE_PY.into(), + }]; + for r in recent { + if r.id == candidate.id || same_miner(candidate, r) { + continue; + } + if Some(r.architecture_py.as_str()) == exempt_arch { + continue; + } + let label = if r.id.len() >= 8 { + format!("subm:{}", &r.id[..8]) + } else { + format!("subm:{}", r.id) + }; + v.push(CorpusEntry { + id: label, + source: r.architecture_py.clone(), + }); + } + v +} + +/// Ephemeral candidate row for precheck corpus filtering (not persisted). +#[must_use] +pub fn ephemeral_candidate( + hotkey: &str, + coldkey: Option, + architecture_py: &str, + created_at_ms: u64, +) -> SubmissionState { + SubmissionState { + id: format!("precheck:{hotkey}:{created_at_ms}"), + miner_hotkey: hotkey.to_owned(), + miner_coldkey: coldkey, + epoch: 0, + netuid: 0, + status: prism_store::Stage::Queued, + architecture_py: architecture_py.to_owned(), + training_py: String::new(), + label: None, + pod_id: None, + pod_provider: None, + receipt: None, + metrics_json: None, + bpb: None, + arch_id: None, + review: None, + similarity: None, + final_score: None, + retry_count: 0, + error_detail: None, + created_at_ms, + updated_at_ms: created_at_ms, + } +} + +fn from_hit(hit: &CopyGateHit, quota: PrecheckQuotaView) -> PrecheckResult { + let score = f64::from(hit.similarity_bps) / 10_000.0; + let message = if hit.byte_identical { + "architecture matches an earlier submission byte-for-byte; revise before submitting".into() + } else { + "architecture is AST-near an earlier submission; revise before submitting".into() + }; + PrecheckResult { + similar: true, + verdict: "copied", + matched_against: Some(hit.nearest_id.clone()), + score: Some(score), + message, + byte_identical: Some(hit.byte_identical), + quota, + } +} + +/// Run the intake copy gate against prior art. Training-only callers should +/// short-circuit with [`precheck_skipped`] instead. +#[must_use] +pub fn evaluate_copy_precheck( + candidate: &SubmissionState, + recent: &[SubmissionState], + quota: PrecheckQuotaView, +) -> PrecheckResult { + let corpus = gate_corpus_from_rows(candidate, recent); + match copy_gate( + &candidate.architecture_py, + candidate.created_at_ms, + &corpus, + ) { + Some(hit) => from_hit(&hit, quota), + None => PrecheckResult { + similar: false, + verdict: "clean", + matched_against: None, + score: None, + message: "no earlier architecture copy detected by the pre-LLM gate; full submit still runs similarity + agentic".into(), + byte_identical: None, + quota, + }, + } +} + +/// Training-only / registry-arch rows skip architecture copy comparison. +#[must_use] +pub fn precheck_skipped(quota: PrecheckQuotaView) -> PrecheckResult { + PrecheckResult { + similar: false, + verdict: "skipped", + matched_against: None, + score: None, + message: "training-only entries skip architecture copy precheck (registry arch)".into(), + byte_identical: None, + quota, + } +} + +/// JSON body for a successful precheck response. +#[must_use] +pub fn precheck_json(result: &PrecheckResult) -> Value { + json!(result) +} + +/// JSON body when the daily precheck quota is exhausted. +#[must_use] +pub fn precheck_quota_exceeded_json(quota: &PrecheckQuotaView) -> Value { + json!({ + "error": "precheck daily quota exceeded (3/coldkey/UTC day)", + "code": "precheck_quota_exceeded", + "quota": quota, + "similar": null, + "verdict": null, + "message": "retry after the next UTC day; precheck does not create a submission", + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + use prism_store::{FinalScore, Stage}; + + fn row( + id: &str, + hotkey: &str, + coldkey: Option<&str>, + arch: &str, + created_at_ms: u64, + ) -> SubmissionState { + SubmissionState { + id: id.into(), + miner_hotkey: hotkey.into(), + miner_coldkey: coldkey.map(str::to_owned), + epoch: 1, + netuid: 1, + status: Stage::Terminated, + architecture_py: arch.into(), + training_py: "train".into(), + label: None, + pod_id: None, + pod_provider: None, + receipt: None, + metrics_json: None, + bpb: Some(1.0), + arch_id: None, + review: None, + similarity: None, + final_score: Some(FinalScore::Score(1)), + retry_count: 0, + error_detail: None, + created_at_ms, + updated_at_ms: created_at_ms, + } + } + + #[test] + fn utc_day_epoch() { + assert_eq!(utc_day(0), "1970-01-01"); + assert_eq!(utc_day(86_400), "1970-01-02"); + } + + #[test] + fn same_coldkey_excluded_from_gate_corpus() { + let prior = row("aaaaaaaa", "aa", Some("11"), "arch_a", 1_000); + let next = row("bbbbbbbb", "bb", Some("11"), "arch_b", 2_000); + assert!(gate_corpus_from_rows(&next, &[prior, next.clone()]).is_empty()); + } + + #[test] + fn copy_hit_marks_similar() { + let victim = row( + "aaaaaaaa", + "aa", + Some("11"), + "def build_model(ctx):\n return 1\n", + 1_000, + ); + let copier = ephemeral_candidate( + &"bb".repeat(32), + Some("22".repeat(32)), + "def build_model(ctx):\n return 1\n", + 2_000, + ); + let q = quota_view("2026-08-08", 1, PrecheckIdentityKind::Coldkey); + let r = evaluate_copy_precheck(&copier, &[victim], q); + assert!(r.similar); + assert_eq!(r.verdict, "copied"); + assert_eq!(r.matched_against.as_deref(), Some("subm:aaaaaaaa")); + } +} diff --git a/crates/prism-store/src/dbprism.rs b/crates/prism-store/src/dbprism.rs index ef975874d..597ff7479 100644 --- a/crates/prism-store/src/dbprism.rs +++ b/crates/prism-store/src/dbprism.rs @@ -433,4 +433,24 @@ impl PrismStore for DbPrismStore { arch::fill_arch_meta(&self.pool, &mut out).await?; Ok(out) } + + async fn precheck_quota_get(&self, identity: &str, day: &str) -> Result { + let n = dbs::prism_precheck_quota_get(&self.pool, identity, day) + .await + .map_err(|e| StoreError::Backend(e.to_string()))?; + Ok(u32::try_from(n).unwrap_or(u32::MAX)) + } + + async fn precheck_quota_try_consume( + &self, + identity: &str, + day: &str, + limit: u32, + ) -> Result, StoreError> { + let limit_i = i32::try_from(limit).unwrap_or(i32::MAX); + let n = dbs::prism_precheck_quota_try_consume(&self.pool, identity, day, limit_i) + .await + .map_err(|e| StoreError::Backend(e.to_string()))?; + Ok(n.map(|v| u32::try_from(v).unwrap_or(u32::MAX))) + } } diff --git a/crates/prism-store/src/store.rs b/crates/prism-store/src/store.rs index 2ac82d171..6cb9fbfb9 100644 --- a/crates/prism-store/src/store.rs +++ b/crates/prism-store/src/store.rs @@ -361,6 +361,17 @@ pub trait PrismStore: Send + Sync + std::fmt::Debug { /// Non-terminal rows beyond grace — for the stuck sweep. async fn list_stuck(&self, grace_secs: u64) -> Result, StoreError>; + + /// Precheck attempts used for `(coldkey_or_hotkey, UTC day)` (0 if none). + async fn precheck_quota_get(&self, identity: &str, day: &str) -> Result; + + /// Consume one precheck attempt when under `limit`. `Some(used)` or `None` if full. + async fn precheck_quota_try_consume( + &self, + identity: &str, + day: &str, + limit: u32, + ) -> Result, StoreError>; } /// In-memory store (CI / sim). @@ -375,6 +386,8 @@ pub struct MemoryPrismStore { emitted: Mutex>, /// Emit cursor per netuid (highest fully-submitted leaf epoch). cursors: Mutex>, + /// `(identity, UTC day)` → checks used for similarity precheck. + precheck_quota: Mutex>, } impl MemoryPrismStore { @@ -804,6 +817,37 @@ impl PrismStore for MemoryPrismStore { .cloned() .collect()) } + + async fn precheck_quota_get(&self, identity: &str, day: &str) -> Result { + let map = self + .precheck_quota + .lock() + .map_err(|_| StoreError::Backend("poison".into()))?; + Ok(map + .get(&(identity.to_owned(), day.to_owned())) + .copied() + .unwrap_or(0)) + } + + async fn precheck_quota_try_consume( + &self, + identity: &str, + day: &str, + limit: u32, + ) -> Result, StoreError> { + let mut map = self + .precheck_quota + .lock() + .map_err(|_| StoreError::Backend("poison".into()))?; + let key = (identity.to_owned(), day.to_owned()); + let used = map.get(&key).copied().unwrap_or(0); + if used >= limit { + return Ok(None); + } + let next = used + 1; + map.insert(key, next); + Ok(Some(next)) + } } #[cfg(test)] diff --git a/docs/PRISM.md b/docs/PRISM.md index 6497a9a30..d24a1917f 100644 --- a/docs/PRISM.md +++ b/docs/PRISM.md @@ -183,6 +183,8 @@ in order and terminal-reject with `Score(0)` on hit: from **other miners** (byte hash + `challenge-ast`; same hotkey/coldkey prior art excluded). Byte/AST copy of a **strictly-earlier** submission is rejected. Ties / unknown timestamps fall through; baseline is exempt. + Miners may probe this gate via `POST /v1/submissions/precheck` (quota + 3/coldkey/UTC day) without queuing a submission. 2. **Static source cheat** (`challenge_agentic::static_source_cheat`) — hardcoded `METRICS_JSON=` short-circuit; missing `prism_telemetry.report` / `finish_evaluation` hooks in `training.py`. @@ -236,6 +238,7 @@ audit-only for the bpb score (coherence gate, never a grader). | Route | Purpose | |-------|---------| | `POST /v1/submissions` | Accept a submission (idempotent by `submission_id`); training-only via `arch_id` + `training.py` | +| `POST /v1/submissions/precheck` | Advisory copy-gate on the same payload shape (no queue, no pod, no 1-max spend) | | `GET /v1/submissions` | List (filter `?status=`, `?miner=`) — rows carry `arch_id` | | `GET /v1/submissions/{id}` | Full detail + receipt + scores | | `GET /v1/submissions/{id}/events` | Append-only transition timeline | @@ -246,6 +249,26 @@ audit-only for the bpb score (coherence gate, never a grader). | `GET /v1/recipe/baseline` | Baseline `architecture.py` / `training.py` | | `GET /health` | Liveness | +### Similarity precheck (`POST /v1/submissions/precheck`) + +Miners can dry-run the **pre-LLM copy gate** (byte/AST vs earlier +`architecture.py` from other miners) before burning a real submission. +Auth and payload match submit (JSON or ZIP + `X-Miner-Hotkey`); metagraph +membership is required when the cache is configured. The call does **not** +insert a `prism_submission` row, does **not** mark the 1-max gate, and does +**not** rent a Lium pod or call OpenRouter. + +| Rule | Detail | +|------|--------| +| Logic | Same `copy_gate` + same-hotkey/**same-coldkey** corpus exclusion as intake | +| Quota | **3 attempts per coldkey per UTC day** (hotkey fallback when Owner unknown) — rotating hotkeys does not reset the budget | +| Exhausted | `429` + `code=precheck_quota_exceeded`, `quota.remaining=0` | +| Training-only | `verdict=skipped` (registry arch is copy-exempt by design) | +| Response | `{ similar, verdict, matched_against?, score?, message, quota }` — never returns competitor source | + +`similar: false` / `verdict: clean` is advisory for the cheap gate only; a +real submit still runs static cheat, cheap similarity, and agentic review. + Miners have **full read access to the recipe**: the dataset pin, the budget, the harness semantics listed above, and the baseline sources they may reuse. diff --git a/docs/external-miner/prism.md b/docs/external-miner/prism.md index 8e72c4866..7189eef8d 100644 --- a/docs/external-miner/prism.md +++ b/docs/external-miner/prism.md @@ -101,6 +101,32 @@ architecture is fine, and training-only entries on a published arch are never "copies" by construction. Starting from the published baseline is always allowed. +### Precheck before you submit (recommended) + +Dry-run the same pre-LLM copy gate **without** burning your 1-max slot or a +GPU eval: + +```bash +curl -sS -X POST "$GATEWAY/challenge/prism/v1/submissions/precheck" \ + -H 'content-type: application/zip' \ + -H "X-Miner-Hotkey: $HOTKEY" \ + --data-binary @submission.zip +``` + +| Field | Meaning | +|-------|---------| +| `similar` | `true` → would hard-reject at intake copy gate | +| `verdict` | `clean` / `copied` / `skipped` (training-only) | +| `matched_against` | Corpus id only (never competitor source) | +| `score` | Similarity in `[0,1]` when compared | +| `quota` | `{ day, used, limit: 3, remaining, identity }` | + +**Quota: 3 attempts per coldkey per UTC day** (falls back to hotkey when the +metagraph Owner coldkey is unknown). Rotating hotkeys under the same coldkey +does **not** reset the budget. A 4th call returns `429` / +`precheck_quota_exceeded` with `remaining=0`. Precheck never creates a scored +submission and never rents a Lium pod. + ## Scoring (summary) Final leaf score is pure bits-per-byte (bpb) on the lattice `[0, SCORE_MAX]`. @@ -127,6 +153,7 @@ The global-best model is published to | Route | Use | |-------|-----| +| `POST /v1/submissions/precheck` | Advisory copy-gate (3/coldkey/UTC day); no submit | | `GET /v1/status` | Backend mode, epoch, queue | | `GET /v1/submissions/{id}` | Detail + receipt + scores | | `GET /v1/submissions/{id}/events` | Stage timeline | diff --git a/docs/external-miner/troubleshoot.md b/docs/external-miner/troubleshoot.md index 5c86efa73..8695a30bc 100644 --- a/docs/external-miner/troubleshoot.md +++ b/docs/external-miner/troubleshoot.md @@ -23,6 +23,8 @@ |---------|--------------|---------------| | Rejected submit | Recipe contract | `GET /v1/recipe` + baseline; follow [`PRISM_RECIPE.md`](../PRISM_RECIPE.md) | | Score 0 after review | `Copied` / `Suspicious` | Similarity gate; rewrite; do not paste baseline wholesale | +| `similar: true` on precheck | Would hit intake copy gate | Rewrite `architecture.py`; baseline is fine to start from | +| `429 precheck_quota_exceeded` | 3 prechecks/coldkey/UTC day used | Wait until next UTC day; rotating hotkeys does not reset | | Stuck `Provisioning` | Lium market thinness | Ops-side; watch `GET /v1/jobs` / events | | Idempotent replay | Same `submission_id` | Expected — returns prior row |