diff --git a/README.md b/README.md index f94d3a077..60684fef5 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,7 @@ Most use an LLM as a judge. All of them pick between an **efficient** model and | **[Advisor Gate](docs/routing_algorithms/advisor_gate_routing.md)** | One model serves every turn; a stronger advisor approves its plans and "done" claims, or sends it back. | `advisor` | lifts a weak executor 43.8% → 54.7% | | **[Sub-Agent-Aware](docs/routing_algorithms/subagent_routing.md)** | Delegated sub-agent traffic routes separately from the parent agent. | `subagents` on `passthrough` or `stage_router` | not yet benchmarked | | **[Custom](docs/routing_algorithms/llm_classifier_routing.md#custom-multi-target-routing)** | The first request is judged by an LLM against criteria you define, routing among 2+ of your own models. | `llm_classifier` + `target_selector` policy | not yet benchmarked | +| **[System Prompt Judge](docs/routing_algorithms/system_prompt_judge_routing.md)** | A judge chooses one hidden system prompt from a text DB before a passthrough call. | `system_prompt_judge` | not yet benchmarked | | **[Random](docs/routing_algorithms/random_routing.md)** | Each request is routed at random, uniform or weighted. | `random` | baseline mechanism | Benchmarks are Terminal-Bench 2.1 against a $98.06 Opus 4.8 baseline at 76.0%. diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index fa6c2af5d..26ee4c030 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -16,6 +16,7 @@ pub mod passthrough; pub mod rand; pub mod stage; pub mod subagent; +pub mod system_prompt_judge; pub mod util; diff --git a/crates/libsy/src/algorithms/system_prompt_judge.rs b/crates/libsy/src/algorithms/system_prompt_judge.rs new file mode 100644 index 000000000..8fbfb2bd9 --- /dev/null +++ b/crates/libsy/src/algorithms/system_prompt_judge.rs @@ -0,0 +1,586 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Judge-selected system prompt injection from a hidden action database. + +use std::collections::BTreeSet; +use std::sync::Arc; + +use serde::Deserialize; +use serde_json::Value; +use switchyard_protocol::{ + AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, Message, ModelId, OutputParams, + Request, Role, completion_text, +}; + +use super::util::prompts::drop_exact_replay; +use super::util::robustness::{safe_client_error, safe_error_summary}; +use crate::core::algorithm::{Algorithm, Driver, RoutingOutcome}; +use crate::{LibsyError, Result}; + +const ALGORITHM_NAME: &str = "system_prompt_judge"; +const DEFAULT_MAX_OUTPUT_TOKENS: u64 = 64; +const TRAILING_JUDGE_INSTRUCTION: &str = + "Choose a system-prompt action for the next assistant turn. Return only JSON."; + +/// One hidden action the judge may select for system prompt injection. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PromptInjectionAction { + /// Stable action id the judge returns. + pub id: String, + /// System prompt inserted when this action is selected. + pub prompt: String, +} + +impl PromptInjectionAction { + /// Creates one validated action entry. + pub fn new(id: impl Into, prompt: impl Into) -> Result { + let id = id.into(); + let prompt = prompt.into(); + validate_action_id(&id)?; + if prompt.trim().is_empty() { + return Err(algorithm_error(format!( + "system prompt action {id:?} must not have an empty prompt" + ))); + } + Ok(Self { + id, + prompt: prompt.trim().to_string(), + }) + } + + /// Parses a text action DB. + /// + /// The format is intentionally small: + /// + /// ```text + /// [action_id] + /// System prompt text to inject. + /// + /// [other_action] + /// Another system prompt. + /// ``` + pub fn parse_database(source: &str) -> Result> { + let mut actions = Vec::new(); + let mut current_id: Option = None; + let mut current_prompt = String::new(); + let mut seen = BTreeSet::new(); + + for line in source.lines() { + let trimmed = line.trim(); + if let Some(id) = section_id(trimmed) { + flush_action( + &mut actions, + &mut seen, + current_id.take(), + &mut current_prompt, + )?; + current_id = Some(id.to_string()); + continue; + } + if current_id.is_some() { + current_prompt.push_str(line); + current_prompt.push('\n'); + } else if !trimmed.is_empty() && !trimmed.starts_with('#') { + return Err(algorithm_error( + "system prompt action DB content must appear under [action_id] headers", + )); + } + } + flush_action(&mut actions, &mut seen, current_id, &mut current_prompt)?; + if actions.is_empty() { + return Err(algorithm_error( + "system prompt action DB must contain at least one [action_id] section", + )); + } + Ok(actions) + } +} + +/// Runtime knobs for [`SystemPromptJudge`]. +#[derive(Clone, Debug)] +pub struct SystemPromptJudgeConfig { + /// Most completion tokens the judge verdict may use. + pub max_output_tokens: u64, +} + +impl Default for SystemPromptJudgeConfig { + fn default() -> Self { + Self { + max_output_tokens: DEFAULT_MAX_OUTPUT_TOKENS, + } + } +} + +/// Passthrough routing with a judge-selected hidden system prompt. +pub struct SystemPromptJudge { + target: ModelId, + judge_target: ModelId, + actions: Vec, + config: SystemPromptJudgeConfig, +} + +impl SystemPromptJudge { + /// Creates a prompt-injection route. + pub fn new( + target: ModelId, + judge_target: ModelId, + actions: Vec, + config: SystemPromptJudgeConfig, + ) -> Result { + if actions.is_empty() { + return Err(algorithm_error( + "system_prompt_judge requires at least one action", + )); + } + if config.max_output_tokens == 0 { + return Err(algorithm_error("max_output_tokens must be at least 1")); + } + Ok(Self { + target, + judge_target, + actions, + config, + }) + } + + async fn selected_action( + &self, + driver: &Driver, + request: &Request, + ) -> Option<&PromptInjectionAction> { + let response = driver + .call_model(self.judge_request(request), vec![self.judge_target.clone()]) + .await + .inspect_err(|error| { + tracing::warn!( + target: "libsy", + error = %safe_error_summary(error), + "system-prompt judge unavailable; routing without injection" + ); + }) + .ok()?; + let aggregate = response + .llm_response + .into_agg() + .await + .inspect_err(|error| { + tracing::warn!( + target: "libsy", + error = %safe_client_error(error), + "system-prompt judge response failed; routing without injection" + ); + }) + .ok()?; + let action = parse_verdict(&aggregate) + .inspect_err(|error| { + tracing::warn!( + target: "libsy", + error = %safe_error_summary(error), + "system-prompt judge verdict invalid; routing without injection" + ); + }) + .ok()?; + if action.eq_ignore_ascii_case("none") { + return None; + } + self.actions + .iter() + .find(|entry| entry.id == action) + .or_else(|| { + tracing::warn!( + target: "libsy", + action, + "system-prompt judge selected unknown action; routing without injection" + ); + None + }) + } + + fn judge_request(&self, request: &Request) -> Request { + let mut messages = request.llm_request.messages.clone(); + messages.push(Message::text( + Role::User, + TRAILING_JUDGE_INSTRUCTION.to_string(), + )); + Request { + llm_request: LlmRequest { + model: request.llm_request.model.clone(), + instructions: vec![InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: judge_prompt(&self.actions), + }], + }], + messages, + output: OutputParams { + max_output_tokens: Some(self.config.max_output_tokens), + response_format: Some(serde_json::json!({"type": "json_object"})), + }, + ..LlmRequest::default() + }, + raw_request: None, + metadata: request.metadata.clone(), + } + } +} + +#[async_trait::async_trait] +impl Algorithm for SystemPromptJudge { + fn name(&self) -> &str { + ALGORITHM_NAME + } + + async fn route( + self: Arc, + driver: Driver, + mut request: Request, + ) -> Result { + if let Some(action) = self.selected_action(&driver, &request).await { + tracing::info!( + target: "libsy", + action = action.id, + selected_model = %self.target, + "system-prompt judge injecting action" + ); + request.llm_request.instructions.insert( + 0, + InstructionBlock { + role: Role::System, + content: vec![ContentBlock::Text { + text: action.prompt.clone(), + }], + }, + ); + drop_exact_replay(&mut request); + } + Ok(RoutingOutcome::route_to( + self.target.clone(), + Vec::new(), + request, + )) + } +} + +#[derive(Deserialize)] +struct JudgeVerdict { + action: String, +} + +fn section_id(line: &str) -> Option<&str> { + line.strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + .map(str::trim) + .filter(|id| !id.is_empty()) +} + +fn flush_action( + actions: &mut Vec, + seen: &mut BTreeSet, + id: Option, + prompt: &mut String, +) -> Result<()> { + let Some(id) = id else { + return Ok(()); + }; + if !seen.insert(id.clone()) { + return Err(algorithm_error(format!( + "duplicate system prompt action id {id:?}" + ))); + } + actions.push(PromptInjectionAction::new(id, std::mem::take(prompt))?); + Ok(()) +} + +fn validate_action_id(id: &str) -> Result<()> { + if id.eq_ignore_ascii_case("none") { + return Err(algorithm_error( + "system prompt action id 'none' is reserved", + )); + } + if id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.')) + { + return Ok(()); + } + Err(algorithm_error(format!( + "system prompt action id {id:?} may contain only ASCII letters, digits, '.', '_', or '-'" + ))) +} + +fn judge_prompt(actions: &[PromptInjectionAction]) -> String { + let mut prompt = String::from( + "You are a routing judge. Read the agent conversation and decide whether one hidden \ + system-prompt action should be injected into the next assistant call.\n\ + Select an action only when it clearly helps the next turn. Otherwise select none.\n\ + Return exactly one JSON object: {\"action\":\"\"}.\n\n\ + Hidden action DB:\n", + ); + for action in actions { + prompt.push_str("\n["); + prompt.push_str(&action.id); + prompt.push_str("]\n"); + prompt.push_str(&action.prompt); + prompt.push('\n'); + } + prompt +} + +fn parse_verdict(response: &AggLlmResponse) -> Result { + let completion = completion_text(response); + let text = strip_json_fence(completion.trim()); + let verdict: JudgeVerdict = + serde_json::from_str(text).or_else(|_| parse_action_from_value(text))?; + let action = verdict.action.trim().to_string(); + if action.is_empty() { + return Err(algorithm_error("system-prompt judge returned empty action")); + } + Ok(action) +} + +fn parse_action_from_value(text: &str) -> Result { + let value: Value = serde_json::from_str(text).map_err(|error| { + algorithm_error(format!( + "system-prompt judge reply did not parse as JSON: {error}" + )) + })?; + let action = value + .get("action") + .and_then(Value::as_str) + .ok_or_else(|| algorithm_error("system-prompt judge JSON must contain string action"))?; + Ok(JudgeVerdict { + action: action.to_string(), + }) +} + +fn strip_json_fence(text: &str) -> &str { + let Some(rest) = text.strip_prefix("```") else { + return text; + }; + let rest = rest.strip_prefix("json").unwrap_or(rest); + let rest = rest.trim_start_matches(['\n', '\r']); + rest.strip_suffix("```").map(str::trim).unwrap_or(rest) +} + +fn algorithm_error(message: impl Into) -> LibsyError { + LibsyError::AlgorithmError { + message: message.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use parking_lot::Mutex; + use switchyard_protocol::{LlmResponse, Response, text_request, text_response}; + + use crate::core::testing::{reply, test_drive}; + + #[test] + fn parses_section_based_action_database() -> Result<()> { + let actions = PromptInjectionAction::parse_database( + r#" +# comments before the first section are ignored +[compile_failure] +Focus on the exact compiler error. + +[stuck-loop] +Stop repeating commands and make a new plan. +"#, + )?; + assert_eq!( + actions, + vec![ + PromptInjectionAction::new( + "compile_failure", + "Focus on the exact compiler error.", + )?, + PromptInjectionAction::new( + "stuck-loop", + "Stop repeating commands and make a new plan.", + )?, + ] + ); + Ok(()) + } + + #[tokio::test] + async fn injects_the_prompt_selected_by_the_judge() -> Result<()> { + let actions = vec![PromptInjectionAction::new( + "compile_failure", + "Focus on compiler diagnostics before editing.", + )?]; + let algorithm: Arc = Arc::new(SystemPromptJudge::new( + "target".into(), + "judge".into(), + actions, + SystemPromptJudgeConfig::default(), + )?); + let calls = Arc::new(Mutex::new(Vec::<(String, Request)>::new())); + let recorder = Arc::clone(&calls); + let mut request = Request { + llm_request: text_request(Some("route".to_string()), "nvcc failed"), + raw_request: None, + metadata: None, + }; + request + .llm_request + .preservation + .requests + .insert("openai_chat".into(), serde_json::json!({"model":"route"})); + + let (selected, _) = test_drive( + algorithm, + request, + move |model: ModelId, request: Request| { + let recorder = Arc::clone(&recorder); + async move { + recorder.lock().push((model.to_string(), request.clone())); + if model == "judge" { + Ok(Response { + llm_response: LlmResponse::Agg(text_response( + None, + r#"{"action":"compile_failure"}"#, + )), + metadata: None, + upstream_headers: Default::default(), + }) + } else { + Ok(reply("ok")) + } + } + }, + ) + .await?; + + assert_eq!(selected, "target"); + let calls = calls.lock(); + assert_eq!( + calls + .iter() + .map(|(model, _)| model.as_str()) + .collect::>(), + ["judge", "target"] + ); + let judge_request = &calls[0].1; + let judge_prompt = judge_request + .llm_request + .instructions + .first() + .and_then(|instruction| instruction.content.first()) + .and_then(|block| match block { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }) + .unwrap_or_default(); + assert!(judge_prompt.contains("[compile_failure]")); + assert!(judge_prompt.contains("Focus on compiler diagnostics before editing.")); + assert_eq!( + judge_request + .llm_request + .messages + .first() + .and_then(|message| message.text_content("|")), + Some("nvcc failed".to_string()) + ); + let target_request = &calls[1].1; + let injected = target_request + .llm_request + .instructions + .first() + .and_then(|instruction| instruction.content.first()) + .and_then(|block| match block { + ContentBlock::Text { text } => Some(text.as_str()), + _ => None, + }); + assert_eq!( + injected, + Some("Focus on compiler diagnostics before editing.") + ); + assert!( + target_request.llm_request.preservation.requests.is_empty(), + "mutated requests must not exact-replay the inbound body" + ); + Ok(()) + } + + #[tokio::test] + async fn none_verdict_leaves_request_untouched() -> Result<()> { + let algorithm: Arc = Arc::new(SystemPromptJudge::new( + "target".into(), + "judge".into(), + vec![PromptInjectionAction::new( + "compile_failure", + "diagnose first", + )?], + SystemPromptJudgeConfig::default(), + )?); + let calls = Arc::new(Mutex::new(Vec::<(String, Request)>::new())); + let recorder = Arc::clone(&calls); + + test_drive( + algorithm, + Request { + llm_request: text_request(Some("route".to_string()), "hello"), + raw_request: None, + metadata: None, + }, + move |model: ModelId, request: Request| { + let recorder = Arc::clone(&recorder); + async move { + recorder.lock().push((model.to_string(), request)); + if model == "judge" { + Ok(reply(r#"{"action":"none"}"#)) + } else { + Ok(reply("ok")) + } + } + }, + ) + .await?; + + let calls = calls.lock(); + assert_eq!(calls[1].0, "target"); + assert!(calls[1].1.llm_request.instructions.is_empty()); + Ok(()) + } + + #[tokio::test] + async fn invalid_judge_reply_fails_open() -> Result<()> { + let algorithm: Arc = Arc::new(SystemPromptJudge::new( + "target".into(), + "judge".into(), + vec![PromptInjectionAction::new( + "compile_failure", + "diagnose first", + )?], + SystemPromptJudgeConfig::default(), + )?); + + let (selected, response) = test_drive( + algorithm, + Request { + llm_request: text_request(Some("route".to_string()), "hello"), + raw_request: None, + metadata: None, + }, + |model: ModelId, _request: Request| async move { + if model == "judge" { + Ok(reply("not json")) + } else { + Ok(reply("ok")) + } + }, + ) + .await?; + + assert_eq!(selected, "target"); + assert_eq!( + completion_text(response.llm_response.as_agg().unwrap()), + "ok" + ); + Ok(()) + } +} diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 6e84c456a..771337585 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -28,6 +28,9 @@ pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::subagent::{SubagentRouter, SubagentRouterConfig}; +pub use algorithms::system_prompt_judge::{ + PromptInjectionAction, SystemPromptJudge, SystemPromptJudgeConfig, +}; pub use algorithms::util::affinity::{AffinityRouter, ClassifyTrigger}; pub use algorithms::util::classifier_contract::{ ClassifierContractConfig, ClassifierResponseFormat, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index c0630f199..db427993e 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -6,15 +6,17 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::error::Error; use std::fmt::{Display, Formatter}; -use std::path::PathBuf; +use std::fs; +use std::path::{Path, PathBuf}; use std::sync::Arc; use libsy::{ AdvisorGate, AdvisorGateConfig, Algorithm, ClassifierContractConfig, ClassifierResponseFormat, ClassifyTrigger, CompositeRouter, CompositeRouterConfig, CustomClassifierConfig, CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, - StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, + LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, + PromptInjectionAction, Random, StageRouter, StageRouterConfig, SubagentRouter, + SubagentRouterConfig, SystemPromptJudge, SystemPromptJudgeConfig, TaskClassifierConfig, ToolSemantics, }; use serde::Deserialize; @@ -412,6 +414,18 @@ pub enum AlgorithmSpec { #[serde(default = "default_fail_open")] fail_open: bool, }, + /// A judge chooses a hidden system prompt from a text DB before a passthrough call. + SystemPromptJudge { + /// Target that serves the caller-visible request. + target: String, + /// Judge target that chooses one action id or `none`. + judge_target: String, + /// Text database of `[action_id]` sections containing prompts to inject. + db_path: PathBuf, + /// Most output tokens one judge verdict may use. + #[serde(default = "default_system_prompt_judge_max_tokens")] + max_output_tokens: u64, + }, /// Routes using a checkpoint-backed prefill classifier. PrefillRouter { /// Target names in checkpoint output order. @@ -585,6 +599,7 @@ impl AlgorithmSpec { Self::Advisor { executor_target, .. } => vec![executor_target], + Self::SystemPromptJudge { target, .. } => vec![target], Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(), } } @@ -621,6 +636,7 @@ impl AlgorithmSpec { names.push(&classifier.target); } Self::Advisor { advisor_target, .. } => names.push(advisor_target), + Self::SystemPromptJudge { judge_target, .. } => names.push(judge_target), _ => {} } // A sub-agent classifier calls its own judge, which is never a completion target. @@ -705,6 +721,14 @@ impl AlgorithmSpec { (Category::Any, vec![executor_target.clone()]), (Category::Judge, vec![advisor_target.clone()]), ]), + Self::SystemPromptJudge { + target, + judge_target, + .. + } => category_models([ + (Category::Any, vec![target.clone()]), + (Category::Judge, vec![judge_target.clone()]), + ]), }; let subagents = match self { @@ -746,6 +770,7 @@ impl AlgorithmSpec { | Self::Auto { .. } | Self::Composite { .. } | Self::PrefillRouter { .. } => None, + Self::SystemPromptJudge { .. } => None, } } @@ -755,7 +780,16 @@ impl AlgorithmSpec { context: &str, targets: &BTreeMap, ) -> AlgorithmResult> { - build_algorithm(context, self, targets) + self.build_with_base_dir(context, targets, None) + } + + pub(crate) fn build_with_base_dir( + &self, + context: &str, + targets: &BTreeMap, + base_dir: Option<&Path>, + ) -> AlgorithmResult> { + build_algorithm(context, self, targets, base_dir) } } @@ -1129,6 +1163,7 @@ fn build_algorithm( route_name: &str, config: &AlgorithmSpec, targets: &BTreeMap, + base_dir: Option<&Path>, ) -> AlgorithmResult> { match config { AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})), @@ -1358,6 +1393,38 @@ fn build_algorithm( })?; Ok(Arc::new(algorithm)) } + AlgorithmSpec::SystemPromptJudge { + target, + judge_target, + db_path, + max_output_tokens, + .. + } => { + let target = resolve_target_model_id(route_name, target, targets)?; + let judge_target = resolve_target_model_id(route_name, judge_target, targets)?; + let source = read_action_db(route_name, db_path, base_dir)?; + let actions = PromptInjectionAction::parse_database(&source).map_err(|error| { + AlgorithmConfigError::with_source( + format!("system_prompt_judge route {route_name}: {error}"), + error, + ) + })?; + let algorithm = SystemPromptJudge::new( + target, + judge_target, + actions, + SystemPromptJudgeConfig { + max_output_tokens: *max_output_tokens, + }, + ) + .map_err(|error| { + AlgorithmConfigError::with_source( + format!("system_prompt_judge route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } AlgorithmSpec::PrefillRouter { targets: names, checkpoint, @@ -1419,6 +1486,10 @@ const fn default_fail_open() -> bool { true } +const fn default_system_prompt_judge_max_tokens() -> u64 { + 64 +} + fn classifier_contract(prompt: Option<&str>) -> ClassifierContractConfig { prompt.map_or_else(ClassifierContractConfig::default, |prompt| { ClassifierContractConfig::default().with_prompt(prompt) @@ -1449,3 +1520,26 @@ fn resolve_target_model_id( )) }) } + +fn read_action_db( + route_name: &str, + db_path: &Path, + base_dir: Option<&Path>, +) -> AlgorithmResult { + let path = if db_path.is_absolute() { + db_path.to_path_buf() + } else { + base_dir + .map(|base| base.join(db_path)) + .unwrap_or_else(|| db_path.to_path_buf()) + }; + fs::read_to_string(&path).map_err(|error| { + AlgorithmConfigError::with_source( + format!( + "system_prompt_judge route {route_name}: failed to read db_path {}: {error}", + path.display() + ), + error, + ) + }) +} diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index b6f9ca58d..46ad2ef90 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -37,7 +37,8 @@ pub(crate) fn load_runner(path: impl AsRef) -> RunnerResult { error, ) })?; - runner_from_toml(&source).map_err(|error| { + let base_dir = path.parent().map(Path::to_path_buf); + runner_from_toml_with_base(&source, base_dir.as_deref()).map_err(|error| { RunnerError::configuration_source( format!("invalid server config {}: {error}", path.display()), error, @@ -46,10 +47,14 @@ pub(crate) fn load_runner(path: impl AsRef) -> RunnerResult { } pub(crate) fn runner_from_toml(source: &str) -> RunnerResult { + runner_from_toml_with_base(source, None) +} + +fn runner_from_toml_with_base(source: &str, base_dir: Option<&Path>) -> RunnerResult { let config: DeploymentConfig = toml::from_str(source).map_err(|error| { RunnerError::configuration_source(format!("failed to parse TOML: {error}"), error) })?; - config.build() + config.build(base_dir) } #[derive(Debug, Deserialize)] @@ -154,7 +159,7 @@ impl DeploymentConfig { }) } - fn build(self) -> RunnerResult { + fn build(self, base_dir: Option<&Path>) -> RunnerResult { if self.schema_version != SUPPORTED_SCHEMA_VERSION { return Err(RunnerError::configuration(format!( "unsupported schema_version {}; expected {SUPPORTED_SCHEMA_VERSION}", @@ -229,7 +234,7 @@ impl DeploymentConfig { } let algorithm = config .algorithm - .build(route_name, &targets) + .build_with_base_dir(route_name, &targets, base_dir) .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?; let (route_clients, caller_auth) = self.build_route_clients(route_name, config, &clients)?; @@ -1007,6 +1012,66 @@ new = ["send_message"] Ok(()) } + #[test] + fn system_prompt_judge_route_reads_relative_action_db() -> RunnerResult<()> { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_nanos()) + .unwrap_or_default(); + let dir = std::env::temp_dir().join(format!( + "switchyard-system-prompt-judge-{}-{nonce}", + std::process::id() + )); + std::fs::create_dir_all(&dir) + .unwrap_or_else(|error| panic!("create temp config dir: {error}")); + std::fs::write( + dir.join("actions.txt"), + "[compile_failure]\nFocus on compiler diagnostics before editing.\n", + ) + .unwrap_or_else(|error| panic!("write action db: {error}")); + let config_path = dir.join("routes.toml"); + std::fs::write( + &config_path, + r#" +schema_version = 1 + +[llm_clients.primary] +format = "openai_chat" +base_url = "https://example.test/v1" + +[targets.primary] +id = "model/primary" +llm_client = "primary" + +[targets.judge] +id = "model/judge" +llm_client = "primary" + +[routes.agent] +id = "switchyard/agent" +type = "system_prompt_judge" +target = "primary" +judge_target = "judge" +db_path = "actions.txt" +"#, + ) + .unwrap_or_else(|error| panic!("write route config: {error}")); + + let runner = Runner::load(&config_path)?; + + assert_eq!( + runner + .models() + .map(|model| (model.id.as_str(), model.algorithm)) + .collect::>(), + [("switchyard/agent", "system_prompt_judge")] + ); + std::fs::remove_file(config_path).ok(); + std::fs::remove_file(dir.join("actions.txt")).ok(); + std::fs::remove_dir(dir).ok(); + Ok(()) + } + #[test] fn stage_rejects_ambiguous_or_non_additive_tool_semantics() { for (configured, expected) in [ diff --git a/crates/switchyard-server/README.md b/crates/switchyard-server/README.md index 7e3dbb404..7eb421ac1 100644 --- a/crates/switchyard-server/README.md +++ b/crates/switchyard-server/README.md @@ -79,8 +79,8 @@ upstream, and a route's `id` is the model clients send to select that algorithm. Each target references an entry under `llm_clients`. All configured clients use `TranslatingLlmClient`; supported formats are `openai_chat`, `openai_responses`, and `anthropic_messages`. Supported algorithms are `noop`, `random`, `passthrough`, -`llm_classifier`, and `stage_router`. The optional `prefill-router` feature also enables -the experimental `prefill_router`. See its +`llm_classifier`, `stage_router`, and `system_prompt_judge`. The optional `prefill-router` +feature also enables the experimental `prefill_router`. See its [artifact requirements](../../docs/reference/toml_schema.md#prefill_router). An `api_key_env` value names an environment variable. The TOML never contains the secret itself. If omitted, the client sends no authentication. @@ -144,6 +144,13 @@ are required. All configured semantic names use exact ASCII case-insensitive mat handoff notes, per-tier system prompts, and a capability-judge fallback are documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_routing.md). +A `system_prompt_judge` route calls `judge_target` first, asking it to choose +one action from a plain-text DB configured by `db_path`. When the judge returns +a known action id, Switchyard prepends that action's text as a system prompt +before calling `target`. Invalid judge replies and judge failures fail open to +plain passthrough. See +[System-Prompt-Judge Routing](../../docs/routing_algorithms/system_prompt_judge_routing.md). + ## Codex model discovery `GET /v1/models` returns the standard `data` list and an empty Codex `models` list. diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 76ecbc1a1..eadedebda 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -398,7 +398,9 @@ async fn upstream_chat( .map(|(_, group)| group.trim().to_string()) }) }); - let content = if model == "model/classifier" && custom_target_schema { + let content = if model == "model/prompt-judge" { + r#"{"action":"compile_failure"}"#.to_string() + } else if model == "model/classifier" && custom_target_schema { if requests_invalid_verdict { r#"{"decision":{"target":"unknown"}}"#.to_string() } else { @@ -1342,6 +1344,79 @@ fn load_test_config(toml: &str) -> TestResult { Ok(load_server_state(config.path())?) } +#[tokio::test] +async fn system_prompt_judge_route_injects_selected_action_prompt() -> TestResult { + let upstream = MockUpstream::start().await?; + let temp_dir = tempfile::tempdir()?; + std::fs::write( + temp_dir.path().join("actions.txt"), + "[compile_failure]\nFocus on compiler diagnostics before editing.\n", + )?; + let config_path = temp_dir.path().join("routes.toml"); + std::fs::write( + &config_path, + format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.primary] +id = "model/primary" +llm_client = "upstream" + +[targets.judge] +id = "model/prompt-judge" +llm_client = "upstream" + +[routes.agent] +id = "switchyard/agent" +type = "system_prompt_judge" +target = "primary" +judge_target = "judge" +db_path = "actions.txt" +"#, + base_url = upstream.base_url + ), + )?; + let app = build_switchyard_router(load_server_state(&config_path)?); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/agent", + "messages": [{"role": "user", "content": "nvcc failed"}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + let calls = upstream.calls.lock().await.clone(); + assert_eq!( + calls + .iter() + .map(|call| call["model"].as_str().unwrap_or_default()) + .collect::>(), + ["model/prompt-judge", "model/primary"] + ); + assert_eq!( + calls[1]["messages"][0], + json!({ + "role": "system", + "content": "Focus on compiler diagnostics before editing." + }) + ); + assert_eq!( + calls[1]["messages"][1], + json!({"role": "user", "content": "nvcc failed"}) + ); + Ok(()) +} + fn weighted_random_state(base_url: &str, weights: [u32; 2]) -> TestResult { load_test_config(&format!( r#" diff --git a/docs/core_concepts.md b/docs/core_concepts.md index 9688a501d..cd62e190d 100644 --- a/docs/core_concepts.md +++ b/docs/core_concepts.md @@ -78,6 +78,7 @@ route types: | `random` | Selects among targets using optional relative weights. | | `llm_classifier` | Uses a classifier target to choose between weak and strong targets. | | `stage_router` | Uses tool-result and progress signals to choose an efficient or capable target. | +| `system_prompt_judge` | Uses a judge target to choose one hidden system prompt before serving a target. | Strong, weak, capable, and efficient are roles within an algorithm, not fixed properties of a model. The same upstream model can serve different roles in diff --git a/docs/getting_started.md b/docs/getting_started.md index 965312914..3b3f57aa0 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -154,6 +154,7 @@ settings. The Rust server also supports: | [Random](routing_algorithms/random_routing.md) | You need a weighted split for A/B tests or baselines. | `random` | | [LLM classifier](routing_algorithms/llm_classifier_routing.md) | Request content should decide whether to use the weak or strong target. | `llm_classifier` | | [Stage router](routing_algorithms/stage_router_routing.md) | Built-in or configured tool-activity signals should select an efficient or capable target. | `stage_router` | +| [System prompt judge](routing_algorithms/system_prompt_judge_routing.md) | A judge should choose one hidden system prompt from a text DB before a passthrough call. | `system_prompt_judge` | A single TOML file can declare multiple routes. The table key, such as `routes.smart`, is a local configuration name; each route's `id` is exposed as a diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 9b1d97780..a826273cf 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -370,6 +370,21 @@ advisor's plan. See | `reviewer_system_prompt` | No | packaged prompt | Replaces the APPROVE/REDO reviewer prompt. | | `redo_feedback_prefix` | No | packaged prompt | Replaces the text put in front of a REDO plan fed back to the executor. | +### `system_prompt_judge` + +Calls a judge target first, asking it to choose one hidden system prompt from a +plain-text action DB. If the judge chooses a known action id, Switchyard prepends +that prompt as a system instruction before calling `target`. Judge failures, +invalid JSON, `none`, and unknown action ids fail open to passthrough. See +[System-Prompt-Judge Routing](../routing_algorithms/system_prompt_judge_routing.md). + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `target` | Yes | — | Target that serves the caller-visible request. | +| `judge_target` | Yes | — | Target used to choose one action id or `none`. Not a routing destination. | +| `db_path` | Yes | — | Text DB containing `[action_id]` prompt sections. Relative paths resolve next to the TOML file when loaded from disk. | +| `max_output_tokens` | No | `64` | Maximum completion tokens for the judge verdict. | + ## Validation Errors `--dry-run` prefixes configuration failures with diff --git a/docs/routing_algorithms/overview.md b/docs/routing_algorithms/overview.md index 074b3c151..2707524d4 100644 --- a/docs/routing_algorithms/overview.md +++ b/docs/routing_algorithms/overview.md @@ -20,6 +20,7 @@ configuration and tuning. For the vocabulary these pages use, see | [Composite Routing](composite_routing.md) | Routing algorithms are composed, one setting the configuration of another before handing off. Today an LLM classifier sets a stage router's default tier. | `composite` | | [Escalation-Router Routing](escalation_router_routing.md) | Start every task on the weak tier and escalate to strong when an LLM judge detects trouble. | `llm_classifier` with `escalation` | | [Advisor-Gate Routing](advisor_gate_routing.md) | One model should serve every turn, with a stronger reviewer approving its "done" claims or sending back a redo plan. | `advisor` | +| [System-Prompt-Judge Routing](system_prompt_judge_routing.md) | A judge should choose one hidden system prompt from a text DB before a passthrough call. | `system_prompt_judge` | ## Common route shape diff --git a/docs/routing_algorithms/system_prompt_judge_routing.md b/docs/routing_algorithms/system_prompt_judge_routing.md new file mode 100644 index 000000000..3b981612d --- /dev/null +++ b/docs/routing_algorithms/system_prompt_judge_routing.md @@ -0,0 +1,67 @@ +# System-Prompt-Judge Routing + +`system_prompt_judge` serves every caller-visible turn with one target, but first +asks a judge model whether to inject one hidden system prompt from a text DB. + +Use it for small experiments where a reviewer should choose from a fixed set of +interventions without exposing those interventions to the served agent unless +one is selected. + +## Configuration + +```toml +[targets.primary] +id = "model/agent" +llm_client = "upstream" + +[targets.judge] +id = "model/judge" +llm_client = "upstream" + +[routes.agent] +id = "switchyard/agent" +type = "system_prompt_judge" +target = "primary" +judge_target = "judge" +db_path = "actions.txt" +``` + +`db_path` is resolved relative to the TOML file when the server loads the +deployment from disk. Absolute paths also work. + +The action DB is a plain text file with one `[action_id]` section per prompt: + +```text +[compile_failure] +Focus on the exact compiler error before editing code. + +[stuck_loop] +Stop repeating the same command. Make a new hypothesis and test it. +``` + +For each request, the judge sees the conversation plus the hidden action DB and +returns: + +```json +{"action":"compile_failure"} +``` + +or: + +```json +{"action":"none"} +``` + +If the judge chooses a known action, Switchyard prepends that action's text as a +system instruction on the call to `target`. If the judge fails, returns invalid +JSON, returns `none`, or selects an unknown action, the route fails open and +sends the original request to `target`. + +## Route keys + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `target` | Yes | — | Target that serves the caller-visible request. | +| `judge_target` | Yes | — | Target used to choose one action id or `none`. Not a routing destination. | +| `db_path` | Yes | — | Text DB containing `[action_id]` prompt sections. Relative paths resolve next to the TOML file. | +| `max_output_tokens` | No | `64` | Maximum completion tokens for the judge verdict. | diff --git a/mkdocs.yml b/mkdocs.yml index 3a9565fb5..fa5da7529 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,6 +34,7 @@ nav: - Composite Routing: routing_algorithms/composite_routing.md - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Advisor-Gate Routing: routing_algorithms/advisor_gate_routing.md + - System-Prompt-Judge Routing: routing_algorithms/system_prompt_judge_routing.md - Operations: - Context-Window Handling: operations/context_window.md - Soak Testing: operations/soak_test.md