From 7f853a7d7931ff6f786c4375a21cee89b26d56a3 Mon Sep 17 00:00:00 2001 From: Taishi Morinaga Date: Thu, 17 Sep 2026 15:27:28 +0000 Subject: [PATCH 1/3] feat(routing): add type_safe_classifier backed by TypeSafe's Jev Adds a new `type_safe_classifier` route type that classifies through TypeSafe's Jev "System One Model" instead of a chat-completion judge. - switchyard-libsy stays I/O-free: the classifier depends only on a new TypeSafeProvider port (algorithms/util/typesafe_provider.rs), mirroring LlmTaskClassifier's FallThrough + AffinityRouter + DefaultCategoryClassifier shape so it fails open the same way. - A new switchyard-typesafe-client crate, a sibling to libsy-llm-client, supplies the runner-owned HTTP implementation that calls POST /v1/systemone. The API key is read only from an environment variable and is redacted from Debug output. - switchyard-runner adds an optional [type_safe_client] table and the type_safe_classifier route type, validated the same way llm_classifier's custom mode is (options/models correspondence, default_target, threshold range), minus the judge-model concept this classifier doesn't have. - Falls back to default_target on a low-confidence, unresolved-label, out-of-range/NaN-confidence, or provider-error verdict. - Adds routing_algorithms/type_safe_classifier_routing.md, a [type_safe_client] / type_safe_classifier reference section in toml_schema.md, and CHANGELOG/README entries. cargo test --workspace, cargo clippy --workspace --all-targets -- -D warnings, and cargo fmt --all -- --check all pass. Verified end to end against the real TypeSafe API from a network-unrestricted environment. Closes #723 Signed-off-by: Taishi Morinaga --- CHANGELOG.md | 8 + Cargo.lock | 15 + Cargo.toml | 2 + README.md | 1 + crates/libsy/src/algorithms.rs | 1 + crates/libsy/src/algorithms/llm_class.rs | 6 +- .../libsy/src/algorithms/type_safe_class.rs | 637 ++++++++++++++++++ crates/libsy/src/algorithms/util.rs | 1 + .../src/algorithms/util/typesafe_provider.rs | 128 ++++ crates/libsy/src/lib.rs | 5 + .../src/runtime.rs | 2 +- crates/switchyard-runner/Cargo.toml | 1 + crates/switchyard-runner/src/algorithm.rs | 162 ++++- crates/switchyard-runner/src/config.rs | 183 ++++- crates/switchyard-runner/tests/route.rs | 2 +- crates/switchyard-typesafe-client/Cargo.toml | 29 + crates/switchyard-typesafe-client/README.md | 77 +++ .../examples/live_check.rs | 63 ++ .../switchyard-typesafe-client/src/error.rs | 48 ++ crates/switchyard-typesafe-client/src/lib.rs | 346 ++++++++++ docs/reference/toml_schema.md | 39 ++ docs/routing_algorithms/overview.md | 1 + .../type_safe_classifier_routing.md | 127 ++++ mkdocs.yml | 1 + 24 files changed, 1871 insertions(+), 14 deletions(-) create mode 100644 crates/libsy/src/algorithms/type_safe_class.rs create mode 100644 crates/libsy/src/algorithms/util/typesafe_provider.rs create mode 100644 crates/switchyard-typesafe-client/Cargo.toml create mode 100644 crates/switchyard-typesafe-client/README.md create mode 100644 crates/switchyard-typesafe-client/examples/live_check.rs create mode 100644 crates/switchyard-typesafe-client/src/error.rs create mode 100644 crates/switchyard-typesafe-client/src/lib.rs create mode 100644 docs/routing_algorithms/type_safe_classifier_routing.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a2349b7..9d266636d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,14 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **TypeSafe classifier routing** — new `type_safe_classifier` route type + routes through TypeSafe's Jev "System One Model" instead of a + chat-completion judge. `switchyard-libsy` stays I/O-free: the classifier + depends only on a new `TypeSafeProvider` port, and a new + `switchyard-typesafe-client` crate supplies the runner-owned HTTP + implementation, injected once per deployment via a `[type_safe_client]` + table. Fails open on a low-confidence, unresolved, or out-of-range verdict, + or a provider error. (#723) - **`timeout_ms` on `[llm_clients.]`** — one deadline covers all attempts, retry delays, and the complete response, including stream reads. Unset leaves the wait unbounded; `0` is rejected. A timeout returns `504` without trying another diff --git a/Cargo.lock b/Cargo.lock index b1cbce1be..9cf17f4b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2432,6 +2432,7 @@ dependencies = [ "switchyard-libsy", "switchyard-llm-client", "switchyard-protocol", + "switchyard-typesafe-client", "thiserror 2.0.18", "tokio", "toml", @@ -2517,6 +2518,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "switchyard-typesafe-client" +version = "0.3.0" +dependencies = [ + "async-trait", + "reqwest", + "serde", + "serde_json", + "switchyard-libsy", + "tokio", + "tracing", + "wiremock", +] + [[package]] name = "syn" version = "2.0.119" diff --git a/Cargo.toml b/Cargo.toml index 4151bc415..1a07d2da2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/switchyard-skill-distillation", "crates/switchyard-soak", "crates/switchyard-translation", + "crates/switchyard-typesafe-client", ] [workspace.package] @@ -49,6 +50,7 @@ switchyard-protocol = { path = "crates/protocol", version = "0.3.0" } switchyard-runner = { path = "crates/switchyard-runner", version = "0.3.0" } switchyard-server = { path = "crates/switchyard-server", version = "0.3.0" } switchyard-translation = { path = "crates/switchyard-translation", version = "0.3.0" } +switchyard-typesafe-client = { path = "crates/switchyard-typesafe-client", version = "0.3.0" } strum_macros = "0.28" thiserror = "2" tokio = { version = "1", features = ["full"] } diff --git a/README.md b/README.md index 2740c8c85..a27b40aa6 100644 --- a/README.md +++ b/README.md @@ -247,6 +247,7 @@ Most use an LLM as a judge. All of them pick between an **efficient** model and | Algorithm | How it decides | Route `type` | Benchmark | |---|---|---|---| | **[Capability](docs/routing_algorithms/llm_classifier_routing.md)** | The first request is judged by an LLM. | `llm_classifier` | 71.2% at $79.32 | +| **[TypeSafe Classifier](docs/routing_algorithms/type_safe_classifier_routing.md)** | The first request is judged by TypeSafe's Jev, a non-generative classifier — no chat-completion judge call. | `type_safe_classifier` | not yet benchmarked | | **[Stage](docs/routing_algorithms/stage_router_routing.md)** | Tool responses are judged by pattern matching or an LLM. | `stage_router` | 72.7% at $68.19 | | **[Capability + Stage](docs/routing_algorithms/composite_routing.md)** | Combines the two above. | `composite` | not yet benchmarked | | **[Escalation](docs/routing_algorithms/escalation_router_routing.md)** | Starts efficient. Responses are judged by an LLM for issues, then escalated. | `llm_classifier` + `mode = "escalation"` | 75.7% at $85.00 | diff --git a/crates/libsy/src/algorithms.rs b/crates/libsy/src/algorithms.rs index fa6c2af5d..531c2095f 100644 --- a/crates/libsy/src/algorithms.rs +++ b/crates/libsy/src/algorithms.rs @@ -16,6 +16,7 @@ pub mod passthrough; pub mod rand; pub mod stage; pub mod subagent; +pub mod type_safe_class; pub mod util; diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index c18086e7f..5a87dfe1f 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -92,7 +92,7 @@ impl TaskClassifierVerdict { /// Selects by reference and clones only what survives — a coding-agent /// conversation carries every tool result, so cloning it whole to keep a window /// would copy the transcript on each judged turn. -fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { +pub(crate) fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { let is_instruction = |message: &Message| matches!(message.role, Role::System | Role::Developer); let mut kept: Vec<&Message> = messages.iter().filter(|m| is_instruction(m)).collect(); let Some(task) = messages.iter().position(|m| m.role == Role::User) else { @@ -149,7 +149,7 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { } /// Keeps the opening task and the latest user follow-up when they differ. -fn task_messages(messages: &[Message]) -> Vec { +pub(crate) fn task_messages(messages: &[Message]) -> Vec { let mut user_messages = messages.iter().filter(|message| message.role == Role::User); let Some(opening_task) = user_messages.next() else { return Vec::new(); @@ -498,7 +498,7 @@ impl JudgePolicy for CustomPolicyRuntime { } /// Builds the affinity router a trigger calls for, if any. -fn affinity_router( +pub(crate) fn affinity_router( trigger: ClassifyTrigger, message_hash_fallback: bool, ) -> Option> { diff --git a/crates/libsy/src/algorithms/type_safe_class.rs b/crates/libsy/src/algorithms/type_safe_class.rs new file mode 100644 index 000000000..000adb965 --- /dev/null +++ b/crates/libsy/src/algorithms/type_safe_class.rs @@ -0,0 +1,637 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! TypeSafe-backed (Jev "System One Model") routing. +//! +//! A non-generative classifier reached through the [`TypeSafeProvider`] port +//! rather than `driver.call_model`: unlike [`crate::LlmTaskClassifier`], the +//! judgment step here is never one of the deployment's own chat-completion +//! targets, because TypeSafe's API does not speak any of the wire formats +//! `libsy-llm-client` supports. `libsy` stays I/O-free by depending only on the +//! [`TypeSafeProvider`] trait; the runner constructs and injects the concrete, +//! HTTP-performing implementation. +//! +//! Structured the same way as [`crate::LlmTaskClassifier`]'s custom mode — an +//! affinity-gated `FallThrough` cascade terminated by [`DefaultCategoryClassifier`] +//! — so this mode fails open identically and slots into a route the same way +//! `llm_classifier` does. + +use std::sync::Arc; + +use async_trait::async_trait; +use switchyard_protocol::{Category, ContentBlock, Message, Request, Response, Role}; + +use super::fall_through::FallThrough; +use super::llm_class::{DefaultCategoryClassifier, affinity_router, task_messages, trim_messages}; +use super::util::affinity::ClassifyTrigger; +use super::util::typesafe_provider::{ + TypeSafeClassifierInput, TypeSafeOption, TypeSafeProvider, TypeSafeVerdict, +}; +use crate::core::algorithm::{Algorithm, Driver}; +use crate::core::classifier::{Classification, Classifier, Score}; +use crate::core::state::State; +use crate::{LibsyError, Result}; + +/// Telemetry label for this algorithm's spans, metrics, and logs. +const ALGORITHM_NAME: &str = "type_safe_task_classifier"; + +/// Instruction sent as the provider's question when the deployment leaves +/// [`TypeSafeClassifierConfig::question`] empty. +const DEFAULT_QUESTION: &str = + "Which of the following best describes what this conversation needs next?"; + +/// Settings for a [`TypeSafeTaskClassifier`] route. +#[derive(Clone, Debug)] +pub struct TypeSafeClassifierConfig { + /// Labeled options offered to the provider, e.g. one per routing tier. Each + /// `label` should parse as a [`Category`] with at least one model configured + /// for this route. + pub options: Vec, + /// Instruction sent as the provider's question. Falls back to a generic + /// tier-selection prompt when empty. + pub question: String, + /// Lowest confidence that is trusted; below it (or on any provider failure, + /// or an unresolved label) the request falls through to `default_target`. + pub base_threshold: f64, + /// Category served when the provider fails, returns an unrecognised label, + /// or answers below `base_threshold`. + pub default_target: Category, + /// How often the classifier re-decides this session's target. + pub classify_trigger: ClassifyTrigger, + /// Uses the first user message as the SessionKey for sticky routing when + /// session metadata is unavailable. Requires `classify_trigger = NewSession`. + pub message_hash_fallback: bool, + /// Trailing conversation turns the provider sees on top of the opening task. + /// `None` (the default) sends the opening task and latest user follow-up only. + pub recent_turn_window: Option, +} + +impl Default for TypeSafeClassifierConfig { + fn default() -> Self { + Self { + options: Vec::new(), + question: String::new(), + base_threshold: 0.0, + default_target: Category::Efficient, + classify_trigger: ClassifyTrigger::default(), + message_hash_fallback: false, + recent_turn_window: None, + } + } +} + +impl TypeSafeClassifierConfig { + fn validate(&self) -> Result<()> { + if self.options.is_empty() { + return Err(LibsyError::AlgorithmError { + message: "type_safe_classifier needs at least one option".to_string(), + }); + } + if !(0.0..=1.0).contains(&self.base_threshold) { + return Err(LibsyError::AlgorithmError { + message: format!( + "base_threshold must be between 0 and 1, got {}", + self.base_threshold + ), + }); + } + if self.message_hash_fallback && self.classify_trigger != ClassifyTrigger::NewSession { + return Err(LibsyError::AlgorithmError { + message: "message_hash_fallback requires classify_trigger = new_session" + .to_string(), + }); + } + Ok(()) + } + + fn question(&self) -> &str { + if self.question.trim().is_empty() { + DEFAULT_QUESTION + } else { + &self.question + } + } +} + +/// Flattens windowed task messages into plain text for a provider's `state`/context +/// field. TypeSafe's System One Model classifies free text, not a chat transcript, so +/// each message becomes one labeled line; non-text content (tool calls and results, +/// media, reasoning) is summarized by kind rather than silently dropped, so the +/// provider can still tell that something happened there even though it never sees +/// provider-private or binary payloads. +fn render_context(messages: &[Message]) -> String { + messages + .iter() + .map(render_message) + .collect::>() + .join("\n\n") +} + +fn render_message(message: &Message) -> String { + let role = role_label(message.role); + let body = message + .content + .iter() + .map(render_block) + .collect::>() + .join(" "); + format!("[{role}] {body}") +} + +fn role_label(role: Role) -> &'static str { + match role { + Role::System => "system", + Role::Developer => "developer", + Role::User => "user", + Role::Assistant => "assistant", + Role::Tool => "tool", + } +} + +fn render_block(block: &ContentBlock) -> String { + match block { + ContentBlock::Text { text } => text.clone(), + ContentBlock::Reasoning { .. } => "(reasoning omitted)".to_string(), + ContentBlock::Image { .. } => "(image)".to_string(), + ContentBlock::Audio { .. } => "(audio)".to_string(), + ContentBlock::Video { .. } => "(video)".to_string(), + ContentBlock::File { .. } => "(file)".to_string(), + ContentBlock::ToolCall(call) => format!("(called tool {})", call.name), + ContentBlock::ToolResult(result) => format!( + "(tool result{})", + if result.is_error == Some(true) { + ", failed" + } else { + "" + } + ), + ContentBlock::Refusal { text } => text.clone(), + ContentBlock::Unknown { .. } => "(unrecognized content)".to_string(), + } +} + +/// Raw provider-backed scorer. +/// +/// Always resolves to a `Classification` — a provider error, an unparseable or +/// unconfigured label, and a low-confidence verdict all become +/// [`Classification::Ambiguous`] so the cascade's [`DefaultCategoryClassifier`] +/// fallback can take over. Nothing here ever returns `Err`, matching +/// [`super::util::llm_judge::JudgeClassifier`]'s fail-open contract. +struct TypeSafeClassifier { + provider: Arc, + config: TypeSafeClassifierConfig, +} + +impl TypeSafeClassifier { + fn fall_open(driver: &Driver, reason_code: &str, extra: serde_json::Value) -> Classification { + let mut evidence = serde_json::json!({ + "source": "fail_open", + "reason_code": reason_code, + }); + if let (Some(evidence), Some(extra)) = (evidence.as_object_mut(), extra.as_object()) { + evidence.extend(extra.clone()); + } + driver.set_evidence_if_empty(evidence); + Classification::Ambiguous(vec![]) + } + + fn to_classification(&self, verdict: &TypeSafeVerdict, driver: &Driver) -> Classification { + // `TypeSafeVerdict::confidence` is documented as "not validated at this + // layer" — this is the caller that promise refers to. A NaN or + // out-of-range value must not silently pass the `< base_threshold` + // check below (NaN compares false against everything, so it would + // otherwise be treated as fully confident). + if !(0.0..=1.0).contains(&verdict.confidence) { + tracing::warn!( + algorithm = ALGORITHM_NAME, + label = %verdict.label, + confidence = verdict.confidence, + "type safe classifier returned an out-of-range confidence; falling through" + ); + return Self::fall_open( + driver, + "invalid_confidence", + serde_json::json!({"label": verdict.label, "confidence": verdict.confidence}), + ); + } + + let resolved = verdict.label.parse::().ok().and_then(|category| { + let target = driver.models_for(&category).first()?.clone(); + Some((category, target)) + }); + + let Some((category, target)) = resolved else { + tracing::warn!( + algorithm = ALGORITHM_NAME, + label = %verdict.label, + "type safe classifier returned an unconfigured label; falling through" + ); + return Self::fall_open( + driver, + "unresolved_label", + serde_json::json!({"label": verdict.label}), + ); + }; + + if verdict.confidence < self.config.base_threshold { + return Self::fall_open( + driver, + "low_confidence", + serde_json::json!({"label": verdict.label, "confidence": verdict.confidence}), + ); + } + + driver.set_evidence_if_empty(serde_json::json!({ + "source": "type_safe_classifier", + "label": verdict.label, + "confidence": verdict.confidence, + })); + Classification::Scores(vec![Score { + target, + confidence: verdict.confidence, + category: Some(category), + }]) + } +} + +#[async_trait] +impl Classifier for TypeSafeClassifier { + async fn score( + &self, + _state: &mut State, + request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + let messages = match self.config.recent_turn_window { + Some(window) => trim_messages(&request.llm_request.messages, window), + None => task_messages(&request.llm_request.messages), + }; + let input = TypeSafeClassifierInput { + question: self.config.question().to_string(), + context: render_context(&messages), + }; + + let verdict = match self.provider.classify(input, &self.config.options).await { + Ok(verdict) => verdict, + Err(error) => { + tracing::warn!( + algorithm = ALGORITHM_NAME, + %error, + "type safe classifier request failed; falling through" + ); + let classification = + Self::fall_open(driver, "provider_error", serde_json::Value::Null); + return Ok((classification, None)); + } + }; + + Ok((self.to_classification(&verdict, driver), None)) + } +} + +/// Routes requests through a TypeSafe (Jev "System One Model") classifier. +pub struct TypeSafeTaskClassifier { + route: FallThrough, + /// Classifier used when this router is embedded in another cascade. + inner: Arc>, +} + +impl TypeSafeTaskClassifier { + /// Builds the classifier described by `config`, scoring through `provider`. + /// + /// # Errors + /// + /// Returns an error when `config` has no options, an out-of-range + /// `base_threshold`, or `message_hash_fallback` without a matching trigger. + pub fn new( + provider: Arc, + config: TypeSafeClassifierConfig, + ) -> Result { + config.validate()?; + let default_target = config.default_target.clone(); + let classify_trigger = config.classify_trigger; + let message_hash_fallback = config.message_hash_fallback; + + let scorer: Arc> = Arc::new(TypeSafeClassifier { provider, config }); + + // Affinity comes first so a retained assignment short-circuits the provider call. + let mut route = FallThrough::::new_with_state().with_name(ALGORITHM_NAME); + if let Some(affinity) = affinity_router(classify_trigger, message_hash_fallback).as_ref() { + route = route + .with_processor(affinity.clone()) + .with_classifier(affinity.clone()); + } + let fallback = DefaultCategoryClassifier(default_target); + let route = route + .with_classifier(scorer.clone()) + .with_classifier(Arc::new(fallback)); + + Ok(Self { + route, + inner: scorer, + }) + } +} + +#[async_trait] +impl Classifier for TypeSafeTaskClassifier { + async fn score( + &self, + state: &mut State, + request: &mut Request, + driver: &Driver, + ) -> Result<(Classification, Option)> { + self.inner.score(state, request, driver).await + } +} + +#[async_trait] +impl Algorithm for TypeSafeTaskClassifier { + fn name(&self) -> &str { + ALGORITHM_NAME + } + + async fn route( + self: Arc, + driver: Driver, + request: Request, + ) -> Result { + self.route.execute(driver, request).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::core::algorithm::RuntimeModels; + use crate::core::testing::empty_driver; + use async_trait::async_trait; + use parking_lot::Mutex; + use switchyard_protocol::{ModelId, text_request}; + + use super::super::util::typesafe_provider::TypeSafeProviderError; + + type ProviderResult = std::result::Result; + + /// A provider whose response is scripted per call. + struct StubProvider { + responses: Mutex>, + } + + impl StubProvider { + fn once(response: ProviderResult) -> Self { + Self { + responses: Mutex::new(vec![response]), + } + } + } + + #[async_trait] + impl TypeSafeProvider for StubProvider { + async fn classify( + &self, + _input: TypeSafeClassifierInput, + _options: &[TypeSafeOption], + ) -> ProviderResult { + self.responses + .lock() + .pop() + .expect("StubProvider called more times than scripted") + } + } + + /// Builds a test `Driver` whose runtime models group every `target` under its + /// paired `category`, preserving `targets` order within each group. Pass the + /// same category more than once to give it several targets. + fn driver_with(category_targets: &[(Category, &str)]) -> Driver { + let mut map: std::collections::HashMap> = + std::collections::HashMap::new(); + for (category, target) in category_targets { + map.entry(category.clone()) + .or_default() + .push(ModelId::from(*target)); + } + Driver::new("test", Arc::new(RuntimeModels::new(map))).0 + } + + fn config(options: Vec, base_threshold: f64) -> TypeSafeClassifierConfig { + TypeSafeClassifierConfig { + options, + base_threshold, + default_target: Category::Efficient, + ..Default::default() + } + } + + fn options() -> Vec { + vec![ + TypeSafeOption::new("capable", "complex, multi-step work"), + TypeSafeOption::new("efficient", "short, simple requests"), + ] + } + + #[tokio::test] + async fn a_confident_verdict_routes_to_its_category() -> Result<()> { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: 0.9, + }))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[(Category::Capable, "strong"), (Category::Efficient, "weak")]); + let mut state = State::default(); + let mut request = Request { + llm_request: text_request(None, "hello"), + raw_request: None, + metadata: None, + }; + + let (classification, _) = classifier.score(&mut state, &mut request, &driver).await?; + assert_eq!( + classification.argmax(false)?.map(|score| score.target), + Some(ModelId::from("strong")) + ); + Ok(()) + } + + /// `TypeSafeTaskClassifier::score` (used when embedded inside another + /// cascade) exposes only the raw provider-backed scorer, matching + /// `LlmTaskClassifier`'s contract — its own affinity/default fallback only + /// runs when the classifier is driven end to end as an [`Algorithm`], via + /// [`Algorithm::route`]. These three tests exercise that full path. + async fn routed_target( + classifier: TypeSafeTaskClassifier, + driver: Driver, + ) -> Result> { + let request = Request { + llm_request: text_request(None, "hello"), + raw_request: None, + metadata: None, + }; + let outcome = Arc::new(classifier).route(driver, request).await?; + Ok(outcome.selected_model_ids.first().cloned()) + } + + #[tokio::test] + async fn a_low_confidence_verdict_falls_through_to_the_default() -> Result<()> { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: 0.2, + }))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[ + (Category::Any, "strong"), + (Category::Any, "weak"), + (Category::Capable, "strong"), + (Category::Efficient, "weak"), + ]); + + assert_eq!( + routed_target(classifier, driver).await?, + Some(ModelId::from("weak")) + ); + Ok(()) + } + + #[tokio::test] + async fn an_unresolved_label_falls_through_to_the_default() -> Result<()> { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "not_a_configured_category".to_string(), + confidence: 0.99, + }))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[ + (Category::Any, "strong"), + (Category::Any, "weak"), + (Category::Capable, "strong"), + (Category::Efficient, "weak"), + ]); + + assert_eq!( + routed_target(classifier, driver).await?, + Some(ModelId::from("weak")) + ); + Ok(()) + } + + #[tokio::test] + async fn a_provider_error_falls_through_to_the_default() -> Result<()> { + let provider = Arc::new(StubProvider::once(Err(TypeSafeProviderError( + "network down".to_string(), + )))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[ + (Category::Any, "strong"), + (Category::Any, "weak"), + (Category::Capable, "strong"), + (Category::Efficient, "weak"), + ]); + + assert_eq!( + routed_target(classifier, driver).await?, + Some(ModelId::from("weak")) + ); + Ok(()) + } + + #[tokio::test] + async fn a_nan_confidence_falls_through_to_the_default() -> Result<()> { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: f64::NAN, + }))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[ + (Category::Any, "strong"), + (Category::Any, "weak"), + (Category::Capable, "strong"), + (Category::Efficient, "weak"), + ]); + + // Without an explicit range check, `NaN < base_threshold` is `false`, so a + // naive comparison would treat this as confident enough to route on. + assert_eq!( + routed_target(classifier, driver).await?, + Some(ModelId::from("weak")) + ); + Ok(()) + } + + #[tokio::test] + async fn an_out_of_range_confidence_falls_through_to_the_default() -> Result<()> { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: 1.5, + }))); + let classifier = TypeSafeTaskClassifier::new(provider, config(options(), 0.5))?; + let driver = driver_with(&[ + (Category::Any, "strong"), + (Category::Any, "weak"), + (Category::Capable, "strong"), + (Category::Efficient, "weak"), + ]); + + assert_eq!( + routed_target(classifier, driver).await?, + Some(ModelId::from("weak")) + ); + Ok(()) + } + + #[test] + fn empty_options_are_rejected() { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: 1.0, + }))); + let result = TypeSafeTaskClassifier::new(provider, config(Vec::new(), 0.5)); + assert!(matches!( + result, + Err(LibsyError::AlgorithmError { message }) + if message.contains("at least one option") + )); + } + + #[test] + fn out_of_range_threshold_is_rejected() { + let provider = Arc::new(StubProvider::once(Ok(TypeSafeVerdict { + label: "capable".to_string(), + confidence: 1.0, + }))); + let result = TypeSafeTaskClassifier::new(provider, config(options(), 1.5)); + assert!(matches!( + result, + Err(LibsyError::AlgorithmError { message }) + if message.contains("base_threshold") + )); + } + + #[test] + fn render_context_summarizes_non_text_content() { + use switchyard_protocol::ToolCall; + + let messages = vec![Message { + role: Role::Assistant, + content: vec![ + ContentBlock::Text { + text: "checking logs".to_string(), + }, + ContentBlock::ToolCall(ToolCall { + id: "call-1".to_string(), + name: "grep".to_string(), + arguments: serde_json::json!({}), + }), + ], + }]; + let rendered = render_context(&messages); + assert!(rendered.contains("[assistant]")); + assert!(rendered.contains("checking logs")); + assert!(rendered.contains("(called tool grep)")); + } + + #[test] + fn empty_driver_smoke() { + // Exercises the same test-only constructor `llm_class.rs` relies on, to + // confirm it stays reachable from this sibling module. + let _ = empty_driver(); + } +} diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 0b96ca92a..5272c819a 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -14,6 +14,7 @@ pub(crate) mod target_selector; #[cfg(test)] pub(crate) mod tier_fixtures; pub(crate) mod tool_signals; +pub(crate) mod typesafe_provider; use switchyard_protocol::ModelId; diff --git a/crates/libsy/src/algorithms/util/typesafe_provider.rs b/crates/libsy/src/algorithms/util/typesafe_provider.rs new file mode 100644 index 000000000..535308045 --- /dev/null +++ b/crates/libsy/src/algorithms/util/typesafe_provider.rs @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Port for an externally-judged, non-generative routing decision. +//! +//! This is the seam [`crate::TypeSafeTaskClassifier`] uses to reach a "System One +//! Model" style classifier (for example TypeSafe's Jev) without `libsy` performing +//! any I/O itself. `libsy` depends only on the [`TypeSafeProvider`] trait defined +//! here; the concrete implementation that actually calls out over HTTP lives in a +//! separate, runner-owned crate (see `switchyard-typesafe-client`) and is injected +//! at deployment-load time. +//! +//! This shape directly answers +//! [Switchyard issue #723](https://github.com/NVIDIA-NeMo/Switchyard/issues/723), +//! which asks for such a classifier to be implemented "via an external HTTP +//! judgment step, or a runner-owned classification provider... avoid direct HTTP +//! calls from libsy": the trait is the external judgment step, and the runner owns +//! whatever calls it. + +use async_trait::async_trait; +use std::fmt; + +/// One label a [`TypeSafeProvider`] may return, together with the natural-language +/// criteria describing when it applies. +/// +/// Mirrors the `criteria` map a TypeSafe `Choice` question is asked to pick from: +/// each option's `label` is ordinarily one of the deployment's configured routing +/// categories (`capable`, `efficient`, or a deployment-defined name), and +/// `description` is shown to the provider verbatim. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeSafeOption { + /// The label returned verbatim in [`TypeSafeVerdict::label`] when chosen. + pub label: String, + /// Natural-language criteria shown to the provider for this label. + pub description: String, +} + +impl TypeSafeOption { + /// Builds one labeled option. + pub fn new(label: impl Into, description: impl Into) -> Self { + Self { + label: label.into(), + description: description.into(), + } + } +} + +/// Conversation material handed to a [`TypeSafeProvider`] for one classification call. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TypeSafeClassifierInput { + /// The instruction asked of the provider, such as "which tier does this need?". + pub question: String, + /// Flattened conversation text the provider classifies. Free text, not a chat + /// transcript — a "System One Model" scores state, it does not converse. + pub context: String, +} + +/// A provider's resolved decision: the chosen label and its confidence. +#[derive(Clone, Debug, PartialEq)] +pub struct TypeSafeVerdict { + /// The label the provider chose. + /// + /// Not guaranteed to be one of the labels it was offered, or to parse as a + /// configured routing category — [`crate::TypeSafeTaskClassifier`] treats an + /// unresolved label the same as a missing verdict (fail-open), rather than + /// erroring. + pub label: String, + /// Confidence in `label`, expected in `[0.0, 1.0]`. + /// + /// Not validated at this layer, so a caller comparing it against a threshold + /// should still treat an out-of-range value defensively. + pub confidence: f64, +} + +/// Opaque failure from a [`TypeSafeProvider`] call. +/// +/// [`crate::TypeSafeTaskClassifier`] folds every variant of failure to fail-open — +/// the same request is simply handed to the next classifier in the cascade — so +/// this carries only a display message suitable for logs and telemetry, never +/// structured detail a caller might branch on. +#[derive(Clone, Debug)] +pub struct TypeSafeProviderError(pub String); + +impl fmt::Display for TypeSafeProviderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for TypeSafeProviderError {} + +/// External, non-generative judgment step a `TypeSafeTaskClassifier` delegates to. +/// +/// Implementations perform the actual network call; `libsy` never does. The host +/// (typically `switchyard-runner`, reading credentials from an environment +/// variable — never from TOML) constructs the concrete implementation once per +/// deployment and injects it as `Arc`, so this trait is the +/// entire seam between libsy's routing logic and a provider's API. +#[async_trait] +pub trait TypeSafeProvider: Send + Sync { + /// Classifies `input` against `options`, returning the provider's choice. + /// + /// Implementations should never log or otherwise surface request credentials + /// in the returned error. + async fn classify( + &self, + input: TypeSafeClassifierInput, + options: &[TypeSafeOption], + ) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn option_builder_converts_into_string() { + let option = TypeSafeOption::new("capable", "complex, multi-step work"); + assert_eq!(option.label, "capable"); + assert_eq!(option.description, "complex, multi-step work"); + } + + #[test] + fn provider_error_displays_its_message() { + let error = TypeSafeProviderError("boom".to_string()); + assert_eq!(error.to_string(), "boom"); + } +} diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 6e84c456a..5d75cdff5 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -28,6 +28,7 @@ pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::subagent::{SubagentRouter, SubagentRouterConfig}; +pub use algorithms::type_safe_class::{TypeSafeClassifierConfig, TypeSafeTaskClassifier}; pub use algorithms::util::affinity::{AffinityRouter, ClassifyTrigger}; pub use algorithms::util::classifier_contract::{ ClassifierContractConfig, ClassifierResponseFormat, @@ -36,6 +37,10 @@ pub use algorithms::util::escalation::EscalationJudgeConfig; pub use algorithms::util::prompts::append_note; pub use algorithms::util::subagent::{SubagentGate, SubagentOverride}; pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSemantics, ToolSignals}; +pub use algorithms::util::typesafe_provider::{ + TypeSafeClassifierInput, TypeSafeOption, TypeSafeProvider, TypeSafeProviderError, + TypeSafeVerdict, +}; // Stage-router scoring and tier selection — the shared signal-driven routing // core (scorer, picker, and the `StageClassifier`). diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 0781ab5e8..b92d8a4e8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -740,7 +740,7 @@ mod tests { fn runtime_for(model: &str) -> SwitchyardRuntime { let algorithm = AlgorithmSpec::Noop {} - .build("relay", &BTreeMap::new()) + .build("relay", &BTreeMap::new(), None) .expect("noop route should build"); let route = Route::new( algorithm, diff --git a/crates/switchyard-runner/Cargo.toml b/crates/switchyard-runner/Cargo.toml index 3a80b61b3..3b96fb430 100644 --- a/crates/switchyard-runner/Cargo.toml +++ b/crates/switchyard-runner/Cargo.toml @@ -24,6 +24,7 @@ serde.workspace = true serde_json.workspace = true switchyard-llm-client.workspace = true switchyard-protocol.workspace = true +switchyard-typesafe-client.workspace = true strum_macros.workspace = true thiserror.workspace = true toml = "1.1" diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index b3935a04c..e65c375f4 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -15,7 +15,8 @@ use libsy::{ CustomClassifierPolicy, EscalationJudgeConfig, GateTrigger, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, SubagentRouter, SubagentRouterConfig, TaskClassifierConfig, - ToolSemantics, + ToolSemantics, TypeSafeClassifierConfig, TypeSafeOption, TypeSafeProvider, + TypeSafeTaskClassifier, }; use serde::Deserialize; use switchyard_protocol::{Category, ModelId}; @@ -147,18 +148,46 @@ struct CustomClassifierRouteConfig { pub struct CategoryModelConfig(BTreeMap>); impl CategoryModelConfig { + /// Validates `models` for a judge-backed classifier (`llm_classifier` custom + /// mode), which always calls its judge through `models.judge`. fn validate(&self, route_name: &str, default_target: &Category) -> AlgorithmResult<()> { - for category in [Category::Any, Category::Judge] { + self.validate_for(route_name, "llm_classifier", default_target, true) + } + + /// Validates `models` for a classifier with no judge-model concept of its own + /// (`type_safe_classifier`, whose judgment step is an external provider, not a + /// runtime model). `models.judge` is neither required nor given any special + /// membership exemption here. + fn validate_without_judge( + &self, + route_name: &str, + default_target: &Category, + ) -> AlgorithmResult<()> { + self.validate_for(route_name, "type_safe_classifier", default_target, false) + } + + fn validate_for( + &self, + route_name: &str, + label: &str, + default_target: &Category, + require_judge: bool, + ) -> AlgorithmResult<()> { + let mut required = vec![Category::Any]; + if require_judge { + required.push(Category::Judge); + } + for category in required { if self.get(&category).is_empty() { return Err(AlgorithmConfigError::new(format!( - "llm_classifier route {route_name} models.{} must contain at least one target", + "{label} route {route_name} models.{} must contain at least one target", category.as_str() ))); } } if self.get(default_target).is_empty() { return Err(AlgorithmConfigError::new(format!( - "llm_classifier route {route_name} models.{} must contain at least one target because it is the default_target", + "{label} route {route_name} models.{} must contain at least one target because it is the default_target", default_target.as_str() ))); } @@ -167,12 +196,13 @@ impl CategoryModelConfig { // the selected target against, so a group outside it can never be served. let any = self.get(&Category::Any); for (name, models) in &self.0 { - if name == Category::Judge.as_str() || name == Category::Any.as_str() { + if name == Category::Any.as_str() || (require_judge && name == Category::Judge.as_str()) + { continue; } if let Some(missing) = models.iter().find(|model| !any.contains(model)) { return Err(AlgorithmConfigError::new(format!( - "llm_classifier route {route_name} models.{name} lists target {missing}, which must also appear in models.any" + "{label} route {route_name} models.{name} lists target {missing}, which must also appear in models.any" ))); } } @@ -428,6 +458,41 @@ pub enum AlgorithmSpec { /// Maximum prompts per encoder forward pass. batch_size: Option, }, + /// Routes using a runner-owned TypeSafe (Jev "System One Model") classifier. + /// + /// Unlike `llm_classifier`, the judgment step here never calls one of the + /// deployment's own chat-completion targets: it calls out to an external, + /// non-generative provider through `switchyard-typesafe-client`, which the + /// runner constructs once per deployment from the `[type_safe_client]` + /// table and injects into every route configured this way. A route using + /// this type fails to build when the deployment has no `[type_safe_client]`. + TypeSafeClassifier { + /// Runtime model groups, keyed by category name. `any` is required; the + /// rest follow `llm_classifier` custom mode's shape, minus `judge` — this + /// classifier's judgment step is an external provider, not a runtime model. + models: CategoryModelConfig, + /// Category served when the provider fails, returns an unrecognised + /// label, or answers below `base_threshold`. + default_target: String, + /// Natural-language criteria offered to the provider, keyed by category + /// name. Every key must also be a key of `models` other than `any`. + options: BTreeMap, + /// Instruction sent as the provider's question. Uses a generic + /// tier-selection default when unset. + question: Option, + /// Lowest confidence that is trusted, from 0 to 1. + base_threshold: f64, + /// How often the classifier re-decides this session's target. + #[serde(default)] + classify_trigger: ClassifyTrigger, + /// Uses the first user message as the SessionKey for sticky routing when + /// session metadata is unavailable. Requires `classify_trigger = "new_session"`. + #[serde(default)] + message_hash_fallback: bool, + /// Trailing conversation turns the provider sees on top of the opening + /// task. Unset sends the opening task and latest user follow-up only. + recent_turn_window: Option, + }, } /// What fires an advisor route's review. @@ -586,6 +651,7 @@ impl AlgorithmSpec { executor_target, .. } => vec![executor_target], Self::PrefillRouter { targets, .. } => targets.iter().map(String::as_str).collect(), + Self::TypeSafeClassifier { models, .. } => models.routing_names(), } } @@ -705,6 +771,7 @@ impl AlgorithmSpec { (Category::Any, vec![executor_target.clone()]), (Category::Judge, vec![advisor_target.clone()]), ]), + Self::TypeSafeClassifier { models, .. } => custom_runtime_model_names(models), }; let subagents = match self { @@ -745,7 +812,8 @@ impl AlgorithmSpec { | Self::StageRouter { .. } | Self::Auto { .. } | Self::Composite { .. } - | Self::PrefillRouter { .. } => None, + | Self::PrefillRouter { .. } + | Self::TypeSafeClassifier { .. } => None, } } @@ -754,8 +822,9 @@ impl AlgorithmSpec { &self, context: &str, targets: &BTreeMap, + type_safe_provider: Option<&Arc>, ) -> AlgorithmResult> { - build_algorithm(context, self, targets) + build_algorithm(context, self, targets, type_safe_provider) } } @@ -1128,6 +1197,7 @@ fn build_algorithm( route_name: &str, config: &AlgorithmSpec, targets: &BTreeMap, + type_safe_provider: Option<&Arc>, ) -> AlgorithmResult> { match config { AlgorithmSpec::Noop { .. } => Ok(Arc::new(Noop {})), @@ -1398,6 +1468,82 @@ fn build_algorithm( ))) } } + AlgorithmSpec::TypeSafeClassifier { + models, + default_target, + options, + question, + base_threshold, + classify_trigger, + message_hash_fallback, + recent_turn_window, + } => { + let default_target: Category = default_target.parse().map_err(|error| { + AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} has invalid default_target: {error}" + )) + })?; + if default_target == Category::Judge { + return Err(AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} default_target cannot be judge" + ))); + } + models.validate_without_judge(route_name, &default_target)?; + + if options.is_empty() { + return Err(AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} requires at least one entry in options" + ))); + } + for label in options.keys() { + let category: Category = label.parse().map_err(|error| { + AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} has invalid options key {label:?}: {error}" + )) + })?; + if category == Category::Judge { + return Err(AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} options.{label} cannot be judge \ + (judge is reserved for llm_classifier's chat-completion judge model, not \ + an external provider's routing target)" + ))); + } + if models.get(&category).is_empty() { + return Err(AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} options.{label} has no matching models.{label}" + ))); + } + } + + let provider = type_safe_provider.ok_or_else(|| { + AlgorithmConfigError::new(format!( + "type_safe_classifier route {route_name} requires a [type_safe_client] table in the deployment" + )) + })?; + + let classifier_config = TypeSafeClassifierConfig { + options: options + .iter() + .map(|(label, description)| { + TypeSafeOption::new(label.clone(), description.clone()) + }) + .collect(), + question: question.clone().unwrap_or_default(), + base_threshold: *base_threshold, + default_target, + classify_trigger: *classify_trigger, + message_hash_fallback: *message_hash_fallback, + recent_turn_window: *recent_turn_window, + }; + let algorithm = TypeSafeTaskClassifier::new(Arc::clone(provider), classifier_config) + .map_err(|error| { + AlgorithmConfigError::with_source( + format!("type_safe_classifier route {route_name}: {error}"), + error, + ) + })?; + Ok(Arc::new(algorithm)) + } } } diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 58a8513ee..aee3785c1 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -59,10 +59,30 @@ pub(crate) struct DeploymentConfig { fallback_client: Option, #[serde(default)] llm_clients: BTreeMap, + /// Shared TypeSafe (Jev "System One Model") client settings. Required when any + /// route uses `type = "type_safe_classifier"`; unused otherwise. + #[serde(default)] + type_safe_client: Option, targets: BTreeMap, routes: BTreeMap, } +/// Settings for the single, deployment-wide TypeSafe client. Unlike `llm_clients`, +/// there is at most one of these — every `type_safe_classifier` route shares it. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TypeSafeClientConfig { + /// Environment variable the API key is read from at load time. Never read + /// from TOML, and never logged. + api_key_env: String, + /// Overrides the API root. Defaults to TypeSafe's production endpoint. + #[serde(default)] + base_url: Option, + /// Overrides the model name sent as `model`. Defaults to `"jev-latest"`. + #[serde(default)] + model: Option, +} + #[derive(Debug)] struct RouteConfig { id: ModelId, @@ -212,6 +232,7 @@ impl DeploymentConfig { let clients = self.build_clients(&mut provider_api_keys)?; let targets = self.build_targets(); let fallback_base_url = self.fallback_base_url()?; + let type_safe_provider = self.build_type_safe_provider()?; let mut routes = Vec::with_capacity(self.routes.len()); for (route_name, config) in &self.routes { for target_name in config.callable_target_names() { @@ -229,7 +250,7 @@ impl DeploymentConfig { } let algorithm = config .algorithm - .build(route_name, &targets) + .build(route_name, &targets, type_safe_provider.as_ref()) .map_err(|error| RunnerError::configuration_source(error.to_string(), error))?; let (route_clients, caller_auth) = self.build_route_clients(route_name, config, &clients)?; @@ -459,6 +480,30 @@ impl DeploymentConfig { Ok(Some(config.base_url.as_str().to_string())) } + /// Builds the deployment-wide TypeSafe provider from `[type_safe_client]`, if + /// configured. The API key always comes from the environment, never from + /// this TOML document. + fn build_type_safe_provider(&self) -> RunnerResult>> { + let Some(config) = &self.type_safe_client else { + return Ok(None); + }; + let mut client = switchyard_typesafe_client::TypeSafeHttpClient::from_env( + &config.api_key_env, + ) + .map_err(|error| { + RunnerError::configuration_source(format!("type_safe_client: {error}"), error) + })?; + if let Some(base_url) = &config.base_url { + client = client.with_base_url(base_url.clone()).map_err(|error| { + RunnerError::configuration_source(format!("type_safe_client: {error}"), error) + })?; + } + if let Some(model) = &config.model { + client = client.with_model(model.clone()); + } + Ok(Some(Arc::new(client))) + } + fn build_anthropic_auxiliary_target( &self, route: &RouteConfig, @@ -839,6 +884,142 @@ target = "weak" Ok(()) } + /// Builds a `type_safe_classifier` deployment reusing `VALID_CONFIG`'s `strong`/`weak` + /// targets, with `[type_safe_client]` reading its key from `api_key_env`. + fn type_safe_config(api_key_env: &str) -> String { + format!( + r#"{VALID_CONFIG} + +[type_safe_client] +api_key_env = "{api_key_env}" + +[routes.type_safe] +id = "switchyard/type_safe" +type = "type_safe_classifier" +default_target = "efficient" +base_threshold = 0.5 +options = {{ capable = "complex, multi-step work", efficient = "short, simple requests" }} +models = {{ any = ["strong", "weak"], capable = ["strong"], efficient = ["weak"] }} +"# + ) + } + + #[test] + fn type_safe_classifier_route_builds_and_resolves_its_categories() -> RunnerResult<()> { + const KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_TYPESAFE_KEY"; + unsafe { + std::env::set_var(KEY_ENV, "sk-test"); + } + let result = runner_from_toml(&type_safe_config(KEY_ENV)); + unsafe { + std::env::remove_var(KEY_ENV); + } + let runner = result?; + + let route = runner + .route("switchyard/type_safe") + .expect("type_safe route should exist"); + let models = route.models(); + assert_eq!( + models.models_for(&Category::Capable), + [ModelId::from("strong/model")] + ); + assert_eq!( + models.models_for(&Category::Efficient), + [ModelId::from("weak/model")] + ); + Ok(()) + } + + #[test] + fn type_safe_classifier_without_a_client_table_is_rejected() { + // `type_safe_config` always emits `[type_safe_client]`; drop the whole table to + // exercise a route configured without one. + let config = + type_safe_config("UNUSED").replace("[type_safe_client]\napi_key_env = \"UNUSED\"", ""); + let error = error_message(&config); + assert!( + error.contains("requires a [type_safe_client] table"), + "{error}" + ); + } + + #[test] + fn type_safe_classifier_rejects_an_option_with_no_matching_model_group() { + const KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_TYPESAFE_KEY_UNMATCHED_OPTION"; + unsafe { + std::env::set_var(KEY_ENV, "sk-test"); + } + let config = type_safe_config(KEY_ENV).replace( + r#"options = { capable = "complex, multi-step work", efficient = "short, simple requests" }"#, + r#"options = { capable = "complex, multi-step work", efficient = "short, simple requests", reasoning = "deep analysis" }"#, + ); + let error = error_message(&config); + unsafe { + std::env::remove_var(KEY_ENV); + } + assert!( + error.contains("options.reasoning has no matching models.reasoning"), + "{error}" + ); + } + + #[test] + fn type_safe_classifier_rejects_judge_as_the_default_target() { + const KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_TYPESAFE_KEY_JUDGE_DEFAULT"; + unsafe { + std::env::set_var(KEY_ENV, "sk-test"); + } + let config = type_safe_config(KEY_ENV).replace( + "default_target = \"efficient\"", + "default_target = \"judge\"", + ); + let error = error_message(&config); + unsafe { + std::env::remove_var(KEY_ENV); + } + assert!(error.contains("default_target cannot be judge"), "{error}"); + } + + #[test] + fn type_safe_classifier_rejects_judge_as_an_option() { + const KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_TYPESAFE_KEY_JUDGE_OPTION"; + unsafe { + std::env::set_var(KEY_ENV, "sk-test"); + } + // Also give `judge` a models group, so the failure below is specifically + // the judge-as-target rejection and not "no matching models.judge". + let config = type_safe_config(KEY_ENV) + .replace( + r#"options = { capable = "complex, multi-step work", efficient = "short, simple requests" }"#, + r#"options = { capable = "complex, multi-step work", efficient = "short, simple requests", judge = "n/a" }"#, + ) + .replace( + r#"models = { any = ["strong", "weak"], capable = ["strong"], efficient = ["weak"] }"#, + r#"models = { any = ["strong", "weak"], capable = ["strong"], efficient = ["weak"], judge = ["strong"] }"#, + ); + let error = error_message(&config); + unsafe { + std::env::remove_var(KEY_ENV); + } + assert!(error.contains("options.judge cannot be judge"), "{error}"); + } + + #[test] + fn type_safe_classifier_rejects_an_out_of_range_threshold() { + const KEY_ENV: &str = "SWITCHYARD_CONFIG_TEST_TYPESAFE_KEY_BAD_THRESHOLD"; + unsafe { + std::env::set_var(KEY_ENV, "sk-test"); + } + let config = + type_safe_config(KEY_ENV).replace("base_threshold = 0.5", "base_threshold = 1.5"); + let error = error_message(&config); + unsafe { + std::env::remove_var(KEY_ENV); + } + assert!(error.contains("base_threshold"), "{error}"); + } + #[test] fn duplicate_route_ids_are_rejected() { let config = format!( diff --git a/crates/switchyard-runner/tests/route.rs b/crates/switchyard-runner/tests/route.rs index 68bba5667..7910e568b 100644 --- a/crates/switchyard-runner/tests/route.rs +++ b/crates/switchyard-runner/tests/route.rs @@ -41,7 +41,7 @@ fn plugin_route(client: Arc) -> Route { ModelId::from("semantic-target"), )]); let algorithm = spec - .build("switchyard", &targets) + .build("switchyard", &targets, None) .expect("identity target map should build"); let clients = ClientRouter::new( BTreeMap::from([(ModelId::from("semantic-target"), client)]) diff --git a/crates/switchyard-typesafe-client/Cargo.toml b/crates/switchyard-typesafe-client/Cargo.toml new file mode 100644 index 000000000..df4c9da21 --- /dev/null +++ b/crates/switchyard-typesafe-client/Cargo.toml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-typesafe-client" +version.workspace = true +description = "HTTP client for TypeSafe's System One Model, implementing switchyard-libsy's TypeSafeProvider port" +authors.workspace = true +edition.workspace = true +homepage = "https://github.com/NVIDIA-NeMo/Switchyard" +license.workspace = true +readme = "README.md" +repository.workspace = true +rust-version.workspace = true +documentation = "https://docs.rs/switchyard-typesafe-client" +keywords = ["typesafe", "client", "classifier", "routing"] +publish = ["crates-io"] + +[dependencies] +switchyard-libsy.workspace = true +async-trait.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true + +[dev-dependencies] +tokio.workspace = true +wiremock = "0.6" diff --git a/crates/switchyard-typesafe-client/README.md b/crates/switchyard-typesafe-client/README.md new file mode 100644 index 000000000..2b5bc8c5e --- /dev/null +++ b/crates/switchyard-typesafe-client/README.md @@ -0,0 +1,77 @@ + + +# switchyard-typesafe-client + +An HTTP client for [TypeSafe](https://typesafe.ai)'s "System One Model" (Jev), +implementing `switchyard-libsy`'s [`TypeSafeProvider`] port. + +`switchyard-libsy` stays I/O-free: it depends only on the `TypeSafeProvider` +trait, never on an HTTP client. This crate is the concrete implementation that +actually performs the network call — the "runner-owned classification +provider" described by +[Switchyard issue #723](https://github.com/NVIDIA-NeMo/Switchyard/issues/723). +`switchyard-runner` constructs one [`TypeSafeHttpClient`] per deployment (from +the optional `[type_safe_client]` table in the deployment TOML) and injects it +into every route configured with `type = "type_safe_classifier"`. + +## Usage + +```rust,no_run +use std::sync::Arc; +use switchyard_libsy::{TypeSafeClassifierInput, TypeSafeOption, TypeSafeProvider}; +use switchyard_typesafe_client::TypeSafeHttpClient; + +# async fn example() -> Result<(), Box> { +let client = TypeSafeHttpClient::from_env("TYPESAFE_API_KEY")?; + +let verdict = client + .classify( + TypeSafeClassifierInput { + question: "Which tier does this need?".to_string(), + context: "[user] how do I list files in a directory?".to_string(), + }, + &[ + TypeSafeOption::new("capable", "complex, multi-step work"), + TypeSafeOption::new("efficient", "short, simple requests"), + ], + ) + .await?; + +println!("{} ({:.2})", verdict.label, verdict.confidence); +# Ok(()) +# } +``` + +The API key is always read from an environment variable — [`TypeSafeHttpClient::from_env`] +never accepts one from configuration files, and [`TypeSafeHttpClient`]'s `Debug` +implementation redacts it. + +## Wire format + +`POST {base_url}/v1/systemone`, `Authorization: Bearer `: + +```json +{ + "state": "", + "model": "jev-latest", + "questions": { + "route": { + "type": "choice", + "instructions": "", + "criteria": { "capable": "...", "efficient": "..." } + } + } +} +``` + +```json +{ + "usage": { "input_tokens": 42 }, + "answers": { + "route": { "choice": "efficient", "confidence": 0.87 } + } +} +``` diff --git a/crates/switchyard-typesafe-client/examples/live_check.rs b/crates/switchyard-typesafe-client/examples/live_check.rs new file mode 100644 index 000000000..8cead174e --- /dev/null +++ b/crates/switchyard-typesafe-client/examples/live_check.rs @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Live smoke test against the real TypeSafe `/v1/systemone` API. +//! +//! This is NOT part of the crate's automated test suite (those use `wiremock` and +//! never touch the network). Run it by hand, from an environment that can reach +//! `api.typesafe.ai`, with a real API key: +//! +//! ```text +//! export TYPESAFE_API_KEY=sk-... +//! cargo run -p switchyard-typesafe-client --example live_check +//! ``` +//! +//! It sends one real classification request (the same "capable vs. efficient" +//! shape a `type_safe_classifier` route asks in production) and prints the +//! provider's verdict plus token usage. A non-zero exit means the call failed; +//! the printed error is TypeSafe's own response detail (never the API key). + +use switchyard_libsy::{TypeSafeClassifierInput, TypeSafeOption, TypeSafeProvider}; +use switchyard_typesafe_client::TypeSafeHttpClient; + +#[tokio::main] +async fn main() { + let client = match TypeSafeHttpClient::from_env("TYPESAFE_API_KEY") { + Ok(client) => client, + Err(err) => { + eprintln!("could not build client: {err}"); + std::process::exit(1); + } + }; + + let options = vec![ + TypeSafeOption::new( + "capable", + "The request needs a strong, expensive model: multi-step reasoning, \ + ambiguous or open-ended instructions, or high-stakes correctness.", + ), + TypeSafeOption::new( + "efficient", + "The request is simple and well-specified: a short factual question, \ + a small formatting or lookup task, or routine chit-chat.", + ), + ]; + + let input = TypeSafeClassifierInput { + question: "Which model tier does this conversation need?".to_string(), + context: "user: What's 2+2?\nassistant:".to_string(), + }; + + println!("Calling TypeSafe (model = jev-latest) ..."); + match client.classify(input, &options).await { + Ok(verdict) => { + println!("OK"); + println!(" label: {}", verdict.label); + println!(" confidence: {:.3}", verdict.confidence); + } + Err(err) => { + eprintln!("classify() failed: {err}"); + std::process::exit(1); + } + } +} diff --git a/crates/switchyard-typesafe-client/src/error.rs b/crates/switchyard-typesafe-client/src/error.rs new file mode 100644 index 000000000..74a7b94fd --- /dev/null +++ b/crates/switchyard-typesafe-client/src/error.rs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt; + +/// Errors constructing a [`crate::TypeSafeHttpClient`]. +/// +/// These are configuration-time failures (a missing or empty credential, an +/// unparsable base URL) — never surfaced from `classify`, which folds every +/// runtime failure into `switchyard_libsy::TypeSafeProviderError` instead. +#[derive(Debug)] +pub enum TypeSafeClientError { + /// The named environment variable was not set. + MissingApiKey { + /// The environment variable that was read. + variable: String, + }, + /// The named environment variable was set to an empty (or whitespace-only) value. + EmptyApiKey { + /// The environment variable that was read. + variable: String, + }, + /// The supplied base URL could not be parsed. + InvalidBaseUrl { + /// The value that failed to parse. + base_url: String, + /// The underlying parse error, rendered to text. + reason: String, + }, +} + +impl fmt::Display for TypeSafeClientError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MissingApiKey { variable } => { + write!(f, "environment variable {variable} is not set") + } + Self::EmptyApiKey { variable } => { + write!(f, "environment variable {variable} is empty") + } + Self::InvalidBaseUrl { base_url, reason } => { + write!(f, "base_url {base_url:?} is not a valid URL: {reason}") + } + } + } +} + +impl std::error::Error for TypeSafeClientError {} diff --git a/crates/switchyard-typesafe-client/src/lib.rs b/crates/switchyard-typesafe-client/src/lib.rs new file mode 100644 index 000000000..d6d3205e7 --- /dev/null +++ b/crates/switchyard-typesafe-client/src/lib.rs @@ -0,0 +1,346 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![warn(missing_docs)] +#![doc = include_str!("../README.md")] + +mod error; +pub use error::TypeSafeClientError; + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use switchyard_libsy::{ + TypeSafeClassifierInput, TypeSafeOption, TypeSafeProvider, TypeSafeProviderError, + TypeSafeVerdict, +}; + +/// Production API root. Override with [`TypeSafeHttpClient::with_base_url`] for +/// testing or a regional deployment. +const DEFAULT_BASE_URL: &str = "https://api.typesafe.ai"; +/// Model served by default. Override with [`TypeSafeHttpClient::with_model`]. +const DEFAULT_MODEL: &str = "jev-latest"; +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); +/// Name of the single `questions` entry sent on every request, and read back from +/// `answers` in the response. +const QUESTION_NAME: &str = "route"; + +/// HTTP client for TypeSafe's `/v1/systemone` endpoint (the "System One Model" / +/// Jev), implementing [`TypeSafeProvider`]. +/// +/// Never constructed from configuration file contents: the API key always comes +/// from an environment variable via [`TypeSafeHttpClient::from_env`], and the +/// `Debug` implementation redacts it so a logged client value cannot leak it. +#[derive(Clone)] +pub struct TypeSafeHttpClient { + http: reqwest::Client, + base_url: String, + api_key: String, + model: String, +} + +impl std::fmt::Debug for TypeSafeHttpClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TypeSafeHttpClient") + .field("base_url", &self.base_url) + .field("model", &self.model) + .field("api_key", &"") + .finish() + } +} + +impl TypeSafeHttpClient { + /// Builds a client from an API key already in hand. + /// + /// Prefer [`Self::from_env`] so the key never passes through configuration + /// file parsing. + pub fn new(api_key: impl Into) -> Self { + Self { + http: reqwest::Client::builder() + .timeout(DEFAULT_TIMEOUT) + .build() + .unwrap_or_else(|_| reqwest::Client::new()), + base_url: DEFAULT_BASE_URL.to_string(), + api_key: api_key.into(), + model: DEFAULT_MODEL.to_string(), + } + } + + /// Reads the API key from the named environment variable. + /// + /// # Errors + /// + /// Returns [`TypeSafeClientError::MissingApiKey`] when the variable is unset, + /// and [`TypeSafeClientError::EmptyApiKey`] when it is set to an empty or + /// whitespace-only value. + pub fn from_env(variable: &str) -> Result { + let api_key = std::env::var(variable).map_err(|_| TypeSafeClientError::MissingApiKey { + variable: variable.to_string(), + })?; + if api_key.trim().is_empty() { + return Err(TypeSafeClientError::EmptyApiKey { + variable: variable.to_string(), + }); + } + Ok(Self::new(api_key)) + } + + /// Overrides the API root. Defaults to TypeSafe's production endpoint. + /// + /// # Errors + /// + /// Returns [`TypeSafeClientError::InvalidBaseUrl`] when `base_url` does not + /// parse as a URL. + pub fn with_base_url( + mut self, + base_url: impl Into, + ) -> Result { + let base_url = base_url.into(); + if let Err(error) = reqwest::Url::parse(&base_url) { + return Err(TypeSafeClientError::InvalidBaseUrl { + base_url, + reason: error.to_string(), + }); + } + self.base_url = base_url; + Ok(self) + } + + /// Overrides the model name sent as `model`. Defaults to `"jev-latest"`. + pub fn with_model(mut self, model: impl Into) -> Self { + self.model = model.into(); + self + } + + fn endpoint(&self) -> String { + format!("{}/v1/systemone", self.base_url.trim_end_matches('/')) + } +} + +#[derive(Serialize)] +struct SystemOneRequest<'a> { + state: &'a str, + model: &'a str, + questions: BTreeMap<&'static str, ChoiceQuestion<'a>>, +} + +#[derive(Serialize)] +struct ChoiceQuestion<'a> { + #[serde(rename = "type")] + kind: &'static str, + instructions: &'a str, + criteria: BTreeMap<&'a str, &'a str>, +} + +#[derive(Deserialize, Default)] +struct Usage { + #[serde(default)] + #[allow(dead_code)] + // Not surfaced today; kept so a wire-shape mismatch fails to parse loudly. + input_tokens: u64, +} + +#[derive(Deserialize)] +struct ChoiceAnswer { + choice: String, + confidence: f64, +} + +#[derive(Deserialize)] +struct SystemOneResponse { + #[serde(default)] + #[allow(dead_code)] + usage: Usage, + answers: BTreeMap, +} + +#[async_trait] +impl TypeSafeProvider for TypeSafeHttpClient { + async fn classify( + &self, + input: TypeSafeClassifierInput, + options: &[TypeSafeOption], + ) -> Result { + let criteria = options + .iter() + .map(|option| (option.label.as_str(), option.description.as_str())) + .collect::>(); + let mut questions = BTreeMap::new(); + questions.insert( + QUESTION_NAME, + ChoiceQuestion { + kind: "choice", + instructions: &input.question, + criteria, + }, + ); + let body = SystemOneRequest { + state: &input.context, + model: &self.model, + questions, + }; + + let response = self + .http + .post(self.endpoint()) + .bearer_auth(&self.api_key) + .json(&body) + .send() + .await + .map_err(|error| TypeSafeProviderError(format!("typesafe request failed: {error}")))?; + + let status = response.status(); + if !status.is_success() { + // The body may include diagnostic detail from TypeSafe but never the caller's + // own credentials (those only ever appear in the request's Authorization header, + // which is never echoed back here). + let detail = response.text().await.unwrap_or_default(); + return Err(TypeSafeProviderError(format!( + "typesafe returned HTTP {status}: {detail}" + ))); + } + + let parsed: SystemOneResponse = response.json().await.map_err(|error| { + TypeSafeProviderError(format!("typesafe response was not valid JSON: {error}")) + })?; + + let answer = parsed.answers.get(QUESTION_NAME).ok_or_else(|| { + TypeSafeProviderError(format!( + "typesafe response is missing the {QUESTION_NAME:?} answer" + )) + })?; + + Ok(TypeSafeVerdict { + label: answer.choice.clone(), + confidence: answer.confidence, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use wiremock::matchers::{header, method, path}; + use wiremock::{Mock, MockServer, ResponseTemplate}; + + fn options() -> Vec { + vec![ + TypeSafeOption::new("capable", "complex, multi-step work"), + TypeSafeOption::new("efficient", "short, simple requests"), + ] + } + + fn input() -> TypeSafeClassifierInput { + TypeSafeClassifierInput { + question: "Which tier does this need?".to_string(), + context: "[user] list files in a directory".to_string(), + } + } + + #[test] + fn from_env_reports_a_missing_variable() { + let error = TypeSafeHttpClient::from_env("SWITCHYARD_TYPESAFE_TEST_UNSET_VAR"); + assert!(matches!( + error, + Err(TypeSafeClientError::MissingApiKey { variable }) + if variable == "SWITCHYARD_TYPESAFE_TEST_UNSET_VAR" + )); + } + + #[test] + fn debug_output_redacts_the_api_key() { + let client = TypeSafeHttpClient::new("sk-super-secret"); + let rendered = format!("{client:?}"); + assert!(!rendered.contains("sk-super-secret")); + assert!(rendered.contains("")); + } + + #[test] + fn invalid_base_url_is_rejected() { + let result = TypeSafeHttpClient::new("key").with_base_url("not a url"); + assert!(matches!( + result, + Err(TypeSafeClientError::InvalidBaseUrl { .. }) + )); + } + + #[tokio::test] + async fn classify_parses_a_successful_response() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/systemone")) + .and(header("Authorization", "Bearer sk-test")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "usage": {"input_tokens": 42}, + "answers": {"route": {"choice": "efficient", "confidence": 0.87}} + }))) + .mount(&server) + .await; + + let client = TypeSafeHttpClient::new("sk-test") + .with_base_url(server.uri()) + .expect("mock server URI is valid"); + + let verdict = client + .classify(input(), &options()) + .await + .expect("classify should succeed"); + + assert_eq!(verdict.label, "efficient"); + assert!((verdict.confidence - 0.87).abs() < f64::EPSILON); + } + + #[tokio::test] + async fn classify_surfaces_a_non_success_status_without_leaking_the_key() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/systemone")) + .respond_with(ResponseTemplate::new(401).set_body_string("unauthorized")) + .mount(&server) + .await; + + let client = TypeSafeHttpClient::new("sk-test") + .with_base_url(server.uri()) + .expect("mock server URI is valid"); + + let error = client + .classify(input(), &options()) + .await + .expect_err("classify should fail on HTTP 401"); + + let message = error.to_string(); + assert!(message.contains("401")); + assert!(!message.contains("sk-test")); + } + + #[tokio::test] + async fn classify_errors_when_the_answer_is_missing() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/systemone")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "usage": {"input_tokens": 1}, + "answers": {} + }))) + .mount(&server) + .await; + + let client = TypeSafeHttpClient::new("sk-test") + .with_base_url(server.uri()) + .expect("mock server URI is valid"); + + let error = client + .classify(input(), &options()) + .await + .expect_err("classify should fail when the answer is absent"); + assert!(error.to_string().contains("route")); + } + + #[test] + fn with_model_overrides_the_default() { + let client = TypeSafeHttpClient::new("sk-test").with_model("jev-pinned"); + assert_eq!(client.model, "jev-pinned"); + } +} diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index d5c78c17e..e3b8c4b01 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -97,6 +97,23 @@ targets. The server rejects an Anthropic forwarding route called through an OpenAI endpoint, or an OpenAI forwarding route called through an Anthropic endpoint, before it calls an upstream. +## `[type_safe_client]` + +Optional, deployment-wide settings for [TypeSafe](https://docs.typesafe.ai/introduction)'s +Jev "System One Model". Required when any route uses +`type = "type_safe_classifier"`; unused otherwise. Unlike `[llm_clients.]`, +there is at most one of these per deployment — every `type_safe_classifier` +route shares it. See [TypeSafe Classifier Routing](../routing_algorithms/type_safe_classifier_routing.md). + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `api_key_env` | Yes | — | Name of the environment variable holding the TypeSafe API key. | +| `base_url` | No | `https://api.typesafe.ai` | Overrides the API root. | +| `model` | No | `jev-latest` | Overrides the model name sent as `model`. | + +As with every other credential in this file, the key itself is never read from +TOML and never logged. + ## `[targets.]` | Key | Required | Default | Meaning | @@ -267,6 +284,28 @@ Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies the schema automatically: through the structured-output request in `json_schema` mode, or in the prompt in `json_object` mode. +### `type_safe_classifier` + +Routes using a runner-owned TypeSafe (Jev "System One Model") classifier. +Requires a [`[type_safe_client]`](#type_safe_client) table in the deployment. +See [TypeSafe Classifier Routing](../routing_algorithms/type_safe_classifier_routing.md). + +| Key | Required | Default | Meaning | +|---|:---:|---|---| +| `options` | Yes | — | Labeled criteria offered to TypeSafe, keyed by name. Every key must also be a key of `models`. | +| `default_target` | Yes | — | Group used when TypeSafe fails, returns an unconfigured label, or answers below `base_threshold`. Any group except `judge`. | +| `base_threshold` | Yes | — | Lowest confidence that is trusted, in `[0, 1]`. | +| `question` | No | generic tier-selection prompt | Instruction sent as TypeSafe's `instructions` field. | +| `classify_trigger` | No | `every_request` | When the classifier runs. Same semantics as `llm_classifier`. | +| `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `classify_trigger = "new_session"`. | +| `recent_turn_window` | No | unset | When unset, TypeSafe sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | +| `models.any` | Yes | — | Every selectable completion target. Every other group's targets must also appear here; one that does not is rejected at configuration load. | +| `models.