From d772d16855178549c370760eb5f90c6094e40307 Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Fri, 18 Sep 2026 19:16:47 -0700 Subject: [PATCH 1/2] feat(libsy): cap the classifier judge payload with judge_char_budget Signed-off-by: Arnav Dadarya --- crates/libsy/src/algorithms/llm_class.rs | 514 ++++++++++++++++-- crates/libsy/src/algorithms/util.rs | 47 ++ .../libsy/src/algorithms/util/escalation.rs | 120 ++-- crates/switchyard-py/src/libsy_bindings.rs | 9 + crates/switchyard-runner/src/algorithm.rs | 26 +- crates/switchyard-runner/src/config.rs | 34 ++ docs/reference/toml_schema.md | 3 +- .../escalation_router_routing.md | 4 + .../llm_classifier_routing.md | 1 + switchyard_rust/libsy.py | 13 +- 10 files changed, 693 insertions(+), 78 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 12d9241d0..e06a29906 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -13,7 +13,6 @@ use switchyard_protocol::{Category, ContentBlock, Message, Role}; use super::escalation; use super::fall_through::FallThrough; -use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::{AffinityRouter, ClassifyTrigger}; use super::util::classifier_contract::{ ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, @@ -24,6 +23,10 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; +use super::util::{ + DEFAULT_JUDGE_CHAR_BUDGET, DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, truncate_middle, + validate_judge_char_budget, +}; use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::State; @@ -83,51 +86,77 @@ impl TaskClassifierVerdict { } } -/// Keeps the opening task and the last `recent_turn_window` turns after it. A -/// window of `0` keeps the task alone. +/// Keeps the opening task and the last `recent_turn_window` turns after it, within +/// `budget` characters. A window of `0` keeps the task alone. /// /// Inbound decoders normalize client system and developer content into /// `LlmRequest::instructions`, so it never reaches this list. /// +/// The anchors are kept whatever the budget, since the judge cannot route without the +/// task; the window gets what they leave. A task statement larger than the whole budget +/// is clipped rather than dropped. +/// /// Selects by reference and clones only what survives — a coding-agent /// conversation carries every tool result, so cloning it whole to keep a window /// would copy the transcript on each judged turn. -fn trim_messages(messages: &[Message], recent_turn_window: usize) -> Vec { +fn trim_messages(messages: &[Message], recent_turn_window: usize, budget: usize) -> Vec { let is_instruction = |message: &Message| matches!(message.role, Role::System | Role::Developer); let mut kept: Vec<&Message> = messages.iter().filter(|m| is_instruction(m)).collect(); - let Some(task) = messages.iter().position(|m| m.role == Role::User) else { - return kept.into_iter().cloned().collect(); - }; - kept.push(&messages[task]); - - let tail: Vec<&Message> = messages[task + 1..] - .iter() - .filter(|m| !is_instruction(m)) - .collect(); - kept.extend(&tail[window_start(&tail, recent_turn_window)..]); - kept.into_iter().cloned().collect() + if let Some(task) = messages.iter().position(|m| m.role == Role::User) { + kept.push(&messages[task]); + let anchor_chars: usize = kept.iter().map(|m| message_chars(m)).sum(); + let tail: Vec<&Message> = messages[task + 1..] + .iter() + .filter(|m| !is_instruction(m)) + .collect(); + let window_budget = budget.saturating_sub(anchor_chars); + kept.extend(&tail[window_start(&tail, recent_turn_window, window_budget)..]); + } + let mut kept: Vec = kept.into_iter().cloned().collect(); + // Only the anchors can still overrun: the window was sized to fit what they left. + if payload_chars(&kept) > budget { + clip_to_budget(&mut kept, budget); + } + kept } -/// The first index of the trailing window. +/// The first index of the trailing window: the widest suffix inside both limits. +/// +/// The turn count is the configured window. The character `budget` bounds what the judge +/// is sent: a window is counted in turns, and turn size varies by orders of magnitude, so +/// without it one large tool result would decide the judge's cost for a fixed +/// configuration. A message that does not fit on its own drops with everything older; +/// the newest turns, the evidence a routing judge needs most, are what survive. /// /// Counting messages alone can start the window between an assistant tool call and the /// result answering it, leaving the judge a result whose call id was never introduced. The -/// start therefore moves back to the nearest one that keeps every tool pair whole. +/// start therefore moves to the nearest one that keeps every tool pair whole: back to the +/// call when the turn count splits a pair, forward past the result when the budget does, +/// since widening past the budget is not an option. /// -/// One newest-to-oldest pass carries the ids still waiting for a call. Direction is what -/// makes it correct: ids repeat across a conversation, and in this order a call is only -/// ever seen after the results it could answer, so a later call — already passed — clears -/// nothing. A result whose call sits before the opening task, which trimming never reaches, -/// keeps the set non-empty to the end and falls back to the counted start, so an unpairable -/// result costs one pass and cannot widen the window to the whole conversation. -fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { +/// One newest-to-oldest pass carries the ids still waiting for a call and the running +/// size, so the cost is one visit per message however the limits fall. Direction is what +/// makes the pairing correct: ids repeat across a conversation, and in this order a call is +/// only ever seen after the results it could answer, so a later call — already passed — +/// clears nothing. A result whose call sits before the opening task, which trimming never +/// reaches, keeps the set non-empty to the end and falls back to the counted start, so an +/// unpairable result cannot widen the window to the whole conversation. +fn window_start(tail: &[&Message], recent_turn_window: usize, budget: usize) -> usize { let counted = tail.len().saturating_sub(recent_turn_window); // An empty window holds no result to pair, and the loop below never visits its start. if counted == tail.len() { return counted; } let mut unpaired: HashSet<&str> = HashSet::new(); + let mut chars = 0; + // The widest start seen with every pair whole, held in case the budget binds before + // the counted start is reached. + let mut whole = None; for (start, message) in tail.iter().enumerate().rev() { + chars += message_chars(message); + if chars > budget { + return whole.unwrap_or(tail.len()); + } // Blocks reverse too, so a call answers a result only when it precedes it inside // one message as well as across messages. for block in message.content.iter().rev() { @@ -141,8 +170,11 @@ fn window_start(tail: &[&Message], recent_turn_window: usize) -> usize { _ => {} } } - if start <= counted && unpaired.is_empty() { - return start; + if unpaired.is_empty() { + if start <= counted { + return start; + } + whole = Some(start); } } counted @@ -181,9 +213,77 @@ fn task_messages(messages: &[Message]) -> Vec { .collect() } +/// Characters one message costs a judge. +/// +/// Counts tool traffic as well as visible text: in a coding-agent conversation a single +/// tool result is routinely larger than every text block around it, so measuring text +/// alone would report a payload as small while the judge is billed for all of it. +fn message_chars(message: &Message) -> usize { + message.content.iter().map(block_chars).sum() +} + +/// Characters one content block costs a judge. +fn block_chars(block: &ContentBlock) -> usize { + match block { + ContentBlock::Text { text } | ContentBlock::Refusal { text } => text.chars().count(), + ContentBlock::ToolCall(call) => { + call.name.chars().count() + call.arguments.to_string().chars().count() + } + ContentBlock::ToolResult(result) => result.content.iter().map(block_chars).sum(), + // Reasoning is stripped before the judge is called, so it costs nothing. Media and + // unknown blocks are opaque here; their wire cost is not text. + _ => 0, + } +} + +/// Whether clipping can shrink this block. +/// +/// Tool arguments and results are JSON the judge may need to read as a unit, so they cannot +/// be cut down to fit. +fn is_clippable(block: &ContentBlock) -> bool { + matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) +} + +/// Total judge payload size for a selected message list. +fn payload_chars(messages: &[Message]) -> usize { + messages.iter().map(message_chars).sum() +} + +/// Clips text blocks so an unwindowable payload still fits `budget`. +/// +/// The share is per block rather than per message: one message can carry several text +/// blocks, and a per-message allowance would let each of them spend it in full. +/// +/// Blocks that cannot be clipped are charged first, so what remains is what the text is +/// allowed to spend. When they alone exceed the budget every text block collapses to +/// nothing and the payload still overruns; cutting tool JSON to fit would hand the judge +/// malformed values, which is worse than an oversized prompt. +fn clip_to_budget(messages: &mut [Message], budget: usize) { + let blocks = || messages.iter().flat_map(|message| &message.content); + let clippable = blocks().filter(|block| is_clippable(block)).count(); + if clippable == 0 { + return; + } + let fixed: usize = blocks() + .filter(|block| !is_clippable(block)) + .map(block_chars) + .sum(); + let per_block = budget.saturating_sub(fixed) / clippable; + for block in messages.iter_mut().flat_map(|message| &mut message.content) { + if let ContentBlock::Text { text } | ContentBlock::Refusal { text } = block { + *text = truncate_middle(text, per_block); + } + } +} + /// Selects the task messages shown to capability and custom-schema classifiers. struct TaskInput { recent_turn_window: Option, + /// Character budget for the judge payload. + judge_char_budget: usize, } impl ClassifierInput for TaskInput { @@ -191,8 +291,21 @@ impl ClassifierInput for TaskInput { // The default preserves the whole-task anchor and latest user update. A // configured window widens that to the surrounding conversation. let mut messages = match self.recent_turn_window { - Some(window) => trim_messages(&request.llm_request.messages, window), - None => task_messages(&request.llm_request.messages), + // The routing instruction appended below is part of what the judge is sent, so + // its cost comes out of the budget rather than on top of it. + Some(window) => trim_messages( + &request.llm_request.messages, + window, + self.judge_char_budget + .saturating_sub(TRAILING_ROUTING_INSTRUCTION.chars().count()), + ), + None => { + let mut messages = task_messages(&request.llm_request.messages); + if payload_chars(&messages) > self.judge_char_budget { + clip_to_budget(&mut messages, self.judge_char_budget); + } + messages + } }; // Reasoning is provider-private and not required to classify the task. Some // upstreams also reject an unsigned reasoning item replayed without the @@ -318,6 +431,12 @@ pub struct TaskClassifierConfig { /// `Some(n)` widens that to the client instructions, the opening task, and /// the last `n` turns after it. pub recent_turn_window: Option, + /// Character budget for the judge payload. + /// + /// Bounds what one request can spend on a judge call. With `recent_turn_window` set, + /// the window narrows until the selection fits; without it, an oversized task + /// statement is clipped. + pub judge_char_budget: usize, /// Prompt and verdict contract settings for the classifier judge. pub contract: ClassifierContractConfig, /// Maximum completion tokens available to the classifier verdict. @@ -337,6 +456,8 @@ struct TaskClassifierConfigWire { message_hash_fallback: bool, #[serde(default)] recent_turn_window: Option, + #[serde(default = "default_judge_char_budget")] + judge_char_budget: usize, #[serde(default)] prompt: Option, #[serde(default)] @@ -362,6 +483,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { classify_trigger: wire.classify_trigger, message_hash_fallback: wire.message_hash_fallback, recent_turn_window: wire.recent_turn_window, + judge_char_budget: wire.judge_char_budget, contract, max_output_tokens: wire.max_output_tokens, }) @@ -372,6 +494,10 @@ const fn default_judge_max_output_tokens() -> u64 { DEFAULT_JUDGE_MAX_OUTPUT_TOKENS } +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET +} + impl Default for TaskClassifierConfig { fn default() -> Self { Self { @@ -380,6 +506,7 @@ impl Default for TaskClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, contract: ClassifierContractConfig::default(), max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } @@ -389,6 +516,7 @@ impl Default for TaskClassifierConfig { impl TaskClassifierConfig { /// Validates routing thresholds before the classifier is constructed. fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if !(0.0..=1.0).contains(&self.base_threshold) { return Err(LibsyError::AlgorithmError { message: format!( @@ -462,6 +590,8 @@ pub struct CustomClassifierConfig { pub message_hash_fallback: bool, /// Trailing conversation turns shown to the classifier judge. pub recent_turn_window: Option, + /// Character budget for the judge payload. + pub judge_char_budget: usize, /// Maximum completion tokens available to the classifier verdict. pub max_output_tokens: u64, } @@ -480,11 +610,13 @@ impl CustomClassifierConfig { classify_trigger: ClassifyTrigger::default(), message_hash_fallback: false, recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, max_output_tokens: DEFAULT_JUDGE_MAX_OUTPUT_TOKENS, } } fn validate(&self) -> Result<()> { + validate_judge_char_budget(self.judge_char_budget)?; if self.max_output_tokens == 0 { return Err(LibsyError::AlgorithmError { message: "max_output_tokens must be at least 1".to_string(), @@ -633,6 +765,7 @@ impl LlmTaskClassifier { StructuredJudge::new( TaskInput { recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, }, contract, SerdeDecoder::new(), @@ -661,6 +794,7 @@ impl LlmTaskClassifier { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, max_output_tokens, } = config; let contract = ClassifierContract::from_inner_schema(&prompt, response_schema)?; @@ -671,7 +805,10 @@ impl LlmTaskClassifier { }; let classifier: Arc> = Arc::new(JudgeClassifier::new( StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, contract, JsonSchemaDecoder::new(), JudgeRuntimeConfig::new(max_output_tokens)?, @@ -768,7 +905,7 @@ impl Algorithm for LlmTaskClassifier { #[cfg(test)] mod tests { - use std::collections::HashMap; + use std::collections::{BTreeSet, HashMap}; use std::sync::Arc; use parking_lot::Mutex; @@ -1340,8 +1477,19 @@ mod tests { /// The text of each message a judge with `recent_turn_window` would be sent. /// The no-window case is covered by `capability_judge_builds_a_structured_request`. fn capability_judge(recent_turn_window: Option) -> Result { + capability_judge_with_budget(recent_turn_window, DEFAULT_JUDGE_CHAR_BUDGET) + } + + /// A judge whose windowed payload is capped at `judge_char_budget` characters. + fn capability_judge_with_budget( + recent_turn_window: Option, + judge_char_budget: usize, + ) -> Result { Ok(StructuredJudge::new( - TaskInput { recent_turn_window }, + TaskInput { + recent_turn_window, + judge_char_budget, + }, LlmTaskClassifier::load_capability_contract(&ClassifierContractConfig::default())?, SerdeDecoder::new(), JudgeRuntimeConfig::new(DEFAULT_JUDGE_MAX_OUTPUT_TOKENS)?, @@ -1395,6 +1543,301 @@ mod tests { Ok(()) } + /// Builds a request whose windowed selection the judge will be handed. + fn windowed_request(messages: Vec) -> Request { + Request { + llm_request: LlmRequest { + messages, + ..LlmRequest::default() + }, + raw_request: None, + metadata: None, + } + } + + /// The messages a judge with this window and budget is actually sent. + fn budgeted_messages( + messages: Vec, + window: usize, + budget: usize, + ) -> Result> { + let judge = capability_judge_with_budget(Some(window), budget)?; + Ok(judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages) + } + + /// A window is counted in turns, so one turn carrying a large tool result would + /// otherwise decide the judge's cost for a fixed configuration. The window ends at + /// the first message that does not fit, so the newest evidence survives. + #[test] + fn an_oversized_turn_narrows_the_window_to_fit_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "recent 2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + // The anchors and the newest turns stay; only the oversized older turn goes. + assert!(texts.contains(&"client instructions".to_string())); + assert!(texts.contains(&"initial task".to_string())); + assert!(texts.contains(&"recent 2".to_string())); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// Reasoning is stripped before the judge is called, so a large reasoning block must not + /// narrow the window and cost the judge visible turns it could have kept. + #[test] + fn reasoning_does_not_count_against_the_budget() -> Result<()> { + let messages = vec![ + Message::text(Role::User, "initial task"), + Message { + role: Role::Assistant, + content: vec![ + ContentBlock::Reasoning { + text: "r".repeat(20_000), + signature: None, + details: Vec::new(), + }, + ContentBlock::Text { + text: "older answer".to_string(), + }, + ], + }, + Message::text(Role::User, "recent 1"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(texts.contains(&"older answer".to_string()), "{texts:?}"); + Ok(()) + } + + /// A message larger than the budget on its own cannot be kept. It drops with the + /// older turns, not with the newer ones that fit. + #[test] + fn a_message_larger_than_the_budget_drops_only_itself_and_older_turns() -> Result<()> { + let messages = vec![ + Message::text(Role::User, "initial task"), + Message::text(Role::User, "recent 1"), + Message::text(Role::Assistant, "x".repeat(20_000)), + Message::text(Role::User, "recent 3"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + let texts: Vec = built + .iter() + .filter_map(|message| message.text_content("\n")) + .collect(); + + assert!(texts.contains(&"initial task".to_string()), "{texts:?}"); + assert!(texts.contains(&"recent 3".to_string()), "{texts:?}"); + assert!(!texts.contains(&"recent 1".to_string()), "{texts:?}"); + assert!(!texts.iter().any(|text| text.len() > 10_000), "{texts:?}"); + Ok(()) + } + + /// The budget applies without a window too: the task messages are clipped rather + /// than sent whole. + #[test] + fn the_default_path_is_clipped_to_the_budget() -> Result<()> { + let judge = capability_judge_with_budget(None, 1_000)?; + let messages = vec![ + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "reply"), + Message::text(Role::User, "follow-up"), + ]; + let built = judge + .build_request(&State::default(), &windowed_request(messages)) + .llm_request + .messages; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() == Some("follow-up")), + "{built:?}" + ); + Ok(()) + } + + /// The budget can end the window between a call and its result. Widening back to the + /// call is not an option, so the result is dropped rather than sent orphaned. + #[test] + fn a_result_whose_call_does_not_fit_is_dropped_with_it() -> Result<()> { + let mut bulky_call = tool_call("call-1"); + bulky_call.content = vec![ContentBlock::ToolCall(ToolCall { + id: "call-1".to_string(), + name: "search".to_string(), + arguments: Value::String("y".repeat(20_000)), + })]; + let messages = vec![ + Message::text(Role::User, "initial task"), + bulky_call, + tool_result("call-1"), + Message::text(Role::User, "recent"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!( + !built + .iter() + .flat_map(|message| &message.content) + .any(|block| matches!(block, ContentBlock::ToolResult(_))), + "{built:?}" + ); + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() == Some("recent")), + "{built:?}" + ); + Ok(()) + } + + /// The window is cut at whole tool pairs, so a surviving tool result still has the + /// call that introduced its id. + #[test] + fn narrowing_for_the_budget_keeps_tool_pairs_whole() -> Result<()> { + let mut bulky = tool_result("call-1"); + bulky.content = vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".to_string(), + content: vec![ContentBlock::Text { + text: "y".repeat(20_000), + }], + is_error: None, + })]; + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "initial task"), + tool_call("call-1"), + bulky, + tool_call("call-2"), + tool_result("call-2"), + ]; + let built = budgeted_messages(messages, 4, 5_000)?; + + assert!(payload_chars(&built) <= 5_000, "{}", payload_chars(&built)); + let calls: BTreeSet = built + .iter() + .flat_map(|message| &message.content) + .filter_map(|block| match block { + ContentBlock::ToolCall(call) => Some(call.id.clone()), + _ => None, + }) + .collect(); + for block in built.iter().flat_map(|message| &message.content) { + if let ContentBlock::ToolResult(result) = block { + assert!( + calls.contains(&result.tool_call_id), + "orphaned result {:?} in {calls:?}", + result.tool_call_id + ); + } + } + Ok(()) + } + + /// The anchors survive an empty window, so a task statement larger than the whole + /// budget cannot be dropped and has to be clipped instead. + #[test] + fn an_oversized_task_is_clipped_once_the_window_cannot_shrink() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "z".repeat(40_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 1_000)?; + let task = built + .iter() + .filter_map(|message| message.text_content("\n")) + .find(|text| text.starts_with('z')) + .ok_or_else(|| LibsyError::AlgorithmError { + message: "clipped task missing".to_string(), + })?; + + assert!(payload_chars(&built) <= 1_000, "{}", payload_chars(&built)); + // Clipping is marked, so the judge can tell a trimmed task from a short one. + assert!(task.contains("[trimmed]"), "{task}"); + Ok(()) + } + + /// The routing instruction is appended after selection, so the budget has to cover it: + /// a selection sized to the limit would otherwise ship a payload above the limit. + #[test] + fn the_budget_covers_the_appended_routing_instruction() -> Result<()> { + let messages = vec![ + Message::text(Role::System, "client instructions"), + Message::text(Role::User, "w".repeat(4_000)), + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 600)?; + + // `built` includes the trailing instruction, so this measures the whole payload. + assert!( + built + .iter() + .any(|message| message.text_content("\n").as_deref() + == Some(TRAILING_ROUTING_INSTRUCTION)) + ); + assert!(payload_chars(&built) <= 600, "{}", payload_chars(&built)); + Ok(()) + } + + /// One message can carry several text blocks. A per-message allowance would let each + /// block spend it in full, so the share is per block. + #[test] + fn several_text_blocks_in_one_message_share_the_budget() -> Result<()> { + let crowded = Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "a".repeat(9_000), + }, + ContentBlock::Text { + text: "b".repeat(9_000), + }, + ], + }; + let messages = vec![ + Message::text(Role::System, "client instructions"), + crowded, + Message::text(Role::Assistant, "recent"), + ]; + let built = budgeted_messages(messages, 2, 900)?; + + assert!(payload_chars(&built) <= 900, "{}", payload_chars(&built)); + Ok(()) + } + + /// A budget below the minimum could not fit the routing instruction that every windowed + /// payload ends with, so it is rejected at construction. + #[test] + fn a_judge_char_budget_below_the_minimum_is_rejected() { + use crate::algorithms::util::MIN_JUDGE_CHAR_BUDGET; + + assert!(TRAILING_ROUTING_INSTRUCTION.chars().count() < MIN_JUDGE_CHAR_BUDGET); + let with_budget = |judge_char_budget| TaskClassifierConfig { + judge_char_budget, + ..TaskClassifierConfig::default() + }; + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET - 1).validate().is_err()); + assert!(with_budget(MIN_JUDGE_CHAR_BUDGET).validate().is_ok()); + } + fn tool_call(id: &str) -> Message { Message { role: Role::Assistant, @@ -1429,6 +1872,7 @@ mod tests { }); let input = TaskInput { recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, }; let mut request = Request { llm_request: LlmRequest { @@ -1471,7 +1915,7 @@ mod tests { ]; // The five-message tail begins exactly on the tool result. - let kept = trim_messages(&messages, 5); + let kept = trim_messages(&messages, 5, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, @@ -1503,7 +1947,7 @@ mod tests { ]; // The four-message tail begins on the first result, whose own call sits one earlier. - let kept = trim_messages(&messages, 4); + let kept = trim_messages(&messages, 4, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, @@ -1533,7 +1977,7 @@ mod tests { Message::text(Role::User, "recent 2"), ]; - let kept = trim_messages(&messages, 3); + let kept = trim_messages(&messages, 3, DEFAULT_JUDGE_CHAR_BUDGET); assert_eq!( kept, @@ -1605,6 +2049,7 @@ mod tests { let built = TaskInput { recent_turn_window: Some(10), + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, } .build_messages(&State::default(), &request); @@ -1759,6 +2204,7 @@ mod tests { let judge: CapabilityJudge = StructuredJudge::new( TaskInput { recent_turn_window: None, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, }, contract, SerdeDecoder::new(), diff --git a/crates/libsy/src/algorithms/util.rs b/crates/libsy/src/algorithms/util.rs index 0b96ca92a..b43403ce8 100644 --- a/crates/libsy/src/algorithms/util.rs +++ b/crates/libsy/src/algorithms/util.rs @@ -30,3 +30,50 @@ pub(crate) fn decisive(target: &ModelId) -> Classification { /// Default completion budget for internal classifier and escalation judge calls. pub(crate) const DEFAULT_JUDGE_MAX_OUTPUT_TOKENS: u64 = 4_096; + +/// Default character budget for a judge payload, shared by the escalation judge and the +/// windowed classifier judges so one turn carrying a large tool result cannot decide how +/// much a judge call costs. +pub(crate) const DEFAULT_JUDGE_CHAR_BUDGET: usize = 18_000; + +/// Smallest accepted judge payload budget. It must leave room for the framing every judge +/// adds around the conversation, or that framing alone would exceed the budget. +pub(crate) const MIN_JUDGE_CHAR_BUDGET: usize = 256; + +pub(crate) fn validate_judge_char_budget(budget: usize) -> crate::Result<()> { + if budget < MIN_JUDGE_CHAR_BUDGET { + return Err(crate::LibsyError::AlgorithmError { + message: format!("judge_char_budget must be at least {MIN_JUDGE_CHAR_BUDGET}"), + }); + } + Ok(()) +} + +/// Separator marking where [`truncate_middle`] dropped a message's interior. +pub(crate) const TRIM_MARKER: &str = " ...[trimmed] "; + +/// Keeps the head and tail of `text` within `limit` characters. +/// +/// The head gets two thirds of the surviving budget: for a judge reading agent activity the +/// command or error signature that opens a message carries more signal than its trailing +/// output. Clipping is marked so the judge can tell a trimmed message from a short one. +pub(crate) fn truncate_middle(text: &str, limit: usize) -> String { + let chars: Vec = text.chars().collect(); + if chars.len() <= limit { + return text.to_string(); + } + // Below the marker's own width there is no room to say the text was clipped, so keep + // what fits and drop the marker. Marking anyway would push the result past `limit`, + // which callers budgeting a payload rely on it never doing. + let marker = TRIM_MARKER.chars().count(); + if limit <= marker { + return chars[..limit].iter().collect(); + } + let keep = limit - marker; + let head = keep * 2 / 3; + let tail = keep - head; + let mut out: String = chars[..head].iter().collect(); + out.push_str(TRIM_MARKER); + out.extend(chars[chars.len() - tail..].iter()); + out +} diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index 7f5e3ae20..0a5b42600 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -16,6 +16,7 @@ use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; +use super::{DEFAULT_JUDGE_CHAR_BUDGET, truncate_middle, validate_judge_char_budget}; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; @@ -25,10 +26,7 @@ use switchyard_protocol::Request; const PROMPT_TEMPLATE: &str = include_str!("../../prompts/escalation/prompt.md"); const SCHEMA_TEMPLATE: &str = include_str!("../../prompts/escalation/schema.json"); -/// Separator marking where [`truncate_middle`] dropped a message's interior. -const TRIM_MARKER: &str = " ...[trimmed] "; - -/// Suffix marking a transcript cut off by [`MAX_REQUEST_CHARS`]. +/// Suffix marking a transcript cut off by `judge_char_budget`. const TRUNCATION_SUFFIX: &str = "..."; /// Per-message cap for system and developer anchors, which carry no trajectory signal but @@ -42,9 +40,6 @@ const SYSTEM_CHARS: usize = 1_000; /// thousand characters, so this gets the widest anchor budget. const TASK_CHARS: usize = 4_000; -/// Backstop on the assembled transcript; the per-message caps normally bind first. -const MAX_REQUEST_CHARS: usize = 18_000; - /// The tuning surface for the trajectory judge. /// /// The routing settings retain their benchmarked defaults. Everything else is a fixed invariant @@ -61,6 +56,17 @@ pub struct EscalationJudgeConfig { pub recent_turn_window: usize, /// Per-message cap inside the trailing window. pub window_message_chars: usize, + /// Most characters the assembled transcript may use. The per-message caps normally + /// bind first; this backstop drops the oldest window lines when they do not. + /// + /// Not read from the `escalation` table: hosts set it from the route-level + /// `judge_char_budget`, which applies to every classifier mode. + #[serde(skip, default = "default_judge_char_budget")] + pub judge_char_budget: usize, +} + +const fn default_judge_char_budget() -> usize { + DEFAULT_JUDGE_CHAR_BUDGET } impl EscalationJudgeConfig { @@ -79,7 +85,7 @@ impl EscalationJudgeConfig { self.window_message_chars )); } - Ok(()) + validate_judge_char_budget(self.judge_char_budget) } } @@ -89,6 +95,7 @@ impl Default for EscalationJudgeConfig { confirmations: 2, recent_turn_window: 28, window_message_chars: 500, + judge_char_budget: DEFAULT_JUDGE_CHAR_BUDGET, } } } @@ -243,27 +250,6 @@ fn collect_text(content: &[ContentBlock], parts: &mut Vec) { } } -/// Keeps the head and tail of `text` within `limit` characters. -/// -/// The head gets two thirds of the surviving budget: for a trajectory judge the command or -/// error signature that opens a message carries more signal than its trailing output. -fn truncate_middle(text: &str, limit: usize) -> String { - let chars: Vec = text.chars().collect(); - if chars.len() <= limit { - return text.to_string(); - } - let keep = limit - .saturating_sub(TRIM_MARKER.chars().count()) - .max(20) - .min(chars.len()); - let head = keep * 2 / 3; - let tail = keep - head; - let mut out: String = chars[..head].iter().collect(); - out.push_str(TRIM_MARKER); - out.extend(chars[chars.len() - tail..].iter()); - out -} - /// Renders a compact role-labelled transcript for the judge. /// /// Task-framing user messages are capped individually. System/developer instructions share @@ -271,7 +257,7 @@ fn truncate_middle(text: &str, limit: usize) -> String { /// The trailing window carries recent activity. A coverage header states how much history is /// not shown, so the judge can reason about pace rather than assuming it sees everything. /// -/// When the assembled text still exceeds `max_request_chars`, the oldest window lines go +/// When the assembled text still exceeds `judge_char_budget`, the oldest window lines go /// first: for a trajectory judge the newest evidence is strictly the most valuable. fn summarize_for_judge( instructions: &[InstructionBlock], @@ -340,10 +326,11 @@ fn summarize_for_judge( .join("\n") }; + let budget = config.judge_char_budget; let reserved = assemble(None, &window[window.len().saturating_sub(1)..]) .chars() .count(); - let instruction_budget = MAX_REQUEST_CHARS.saturating_sub(reserved + 1); + let instruction_budget = budget.saturating_sub(reserved + 1); // The remaining budget may be smaller than truncate_middle's minimum retained span. let instruction_text = truncate_middle(&instruction_anchors.join("\n"), instruction_budget) .chars() @@ -351,12 +338,12 @@ fn summarize_for_judge( .collect::(); let instructions = (!instruction_text.is_empty()).then_some(instruction_text.as_str()); let mut text = assemble(instructions, &window); - while text.chars().count() > MAX_REQUEST_CHARS && !window.is_empty() { + while text.chars().count() > budget && !window.is_empty() { window.remove(0); text = assemble(instructions, &window); } - if text.chars().count() > MAX_REQUEST_CHARS { - let keep = MAX_REQUEST_CHARS.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1); + if text.chars().count() > budget { + let keep = budget.saturating_sub(TRUNCATION_SUFFIX.chars().count() + 1); text = text.chars().take(keep).collect::() + TRUNCATION_SUFFIX; } text @@ -406,6 +393,7 @@ mod tests { use switchyard_protocol::{ContentBlock, Message, Role, ToolCall, ToolResult}; use super::*; + use crate::algorithms::util::TRIM_MARKER; use crate::algorithms::util::llm_judge::Judge; fn escalation_judge(max_output_tokens: u64) -> Result { @@ -470,13 +458,13 @@ mod tests { role: Role::Developer, content: Message::text(Role::Developer, "x".repeat(SYSTEM_CHARS)).content, }; - MAX_REQUEST_CHARS / SYSTEM_CHARS + 1 + DEFAULT_JUDGE_CHAR_BUDGET / SYSTEM_CHARS + 1 ]); let built = judge.build_request(&State::default(), &judged); let summary = built.llm_request.messages[0] .text_content("") .expect("summary"); - assert!(summary.chars().count() <= MAX_REQUEST_CHARS); + assert!(summary.chars().count() <= DEFAULT_JUDGE_CHAR_BUDGET); assert!(summary.contains(TRIM_MARKER)); assert!(summary.contains("[system] system constraint")); assert!(summary.contains("[developer] developer constraint")); @@ -554,6 +542,21 @@ mod tests { assert_eq!(truncate_middle("short", 50), "short"); } + /// Callers budget a payload on the promise that clipping never exceeds the limit. Below + /// the marker's own width there is no room to mark the cut, so the marker is dropped + /// rather than pushing the result over. + #[test] + fn truncate_middle_never_exceeds_a_limit_narrower_than_the_marker() { + let text = "a".repeat(100); + for limit in 0..=TRIM_MARKER.chars().count() + 2 { + let trimmed = truncate_middle(&text, limit); + assert!( + trimmed.chars().count() <= limit, + "limit {limit}: {trimmed:?}" + ); + } + } + #[test] fn summary_keeps_anchors_and_the_recent_window() { let mut messages = vec![ @@ -630,8 +633,8 @@ mod tests { #[test] fn summary_drops_oldest_window_lines_under_the_char_cap() { - // MAX_REQUEST_CHARS is a backstop, not a dial: at default settings the window caps - // bind first (28 x 500 plus anchors sits under it), so reaching it takes an unusually + // The budget is a backstop, not a dial: at default settings the window caps bind + // first (28 x 500 plus anchors sits under it), so reaching it takes an unusually // wide per-message cap. That is the point — it only fires on pathological input. let mut messages = vec![ Message::text(Role::System, "framing"), @@ -651,7 +654,7 @@ mod tests { let summary = summarize_for_judge(&[], &messages, 21, &config); assert!( - summary.chars().count() <= MAX_REQUEST_CHARS, + summary.chars().count() <= config.judge_char_budget, "{}", summary.chars().count() ); @@ -661,4 +664,43 @@ mod tests { assert!(summary.contains("19 xxx"), "{summary}"); assert!(!summary.contains("0 xxx"), "{summary}"); } + + /// The cap is the route's `judge_char_budget`, not a constant: a smaller budget drops + /// more of the window. + #[test] + fn the_configured_budget_caps_the_summary() { + let mut messages = vec![Message::text(Role::User, "task")]; + for i in 0..20 { + messages.push(Message::text( + Role::Assistant, + format!("{i} {}", "x".repeat(400)), + )); + } + let config = EscalationJudgeConfig { + judge_char_budget: 2_000, + ..EscalationJudgeConfig::default() + }; + + let summary = summarize_for_judge(&[], &messages, 21, &config); + + assert!( + summary.chars().count() <= 2_000, + "{}", + summary.chars().count() + ); + assert!(summary.contains("19 xxx"), "{summary}"); + assert!(!summary.contains("10 xxx"), "{summary}"); + } + + /// The budget is set by the host from the route-level key, so the `escalation` table + /// neither reads it nor loses the default. + #[test] + fn the_budget_is_not_read_from_the_escalation_table() { + let config: EscalationJudgeConfig = serde_json::from_str("{}").unwrap(); + assert_eq!(config.judge_char_budget, DEFAULT_JUDGE_CHAR_BUDGET); + assert!( + serde_json::from_str::(r#"{"judge_char_budget": 1000}"#) + .is_err() + ); + } } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 9789949a9..3d053c057 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -115,6 +115,7 @@ impl PyEscalationClassifierConfig { confirmations=2, recent_turn_window=28, window_message_chars=500, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -124,6 +125,7 @@ impl PyEscalationClassifierConfig { confirmations: u32, recent_turn_window: usize, window_message_chars: usize, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -134,6 +136,7 @@ impl PyEscalationClassifierConfig { confirmations, recent_turn_window, window_message_chars, + judge_char_budget, }, max_output_tokens, }) @@ -169,6 +172,7 @@ impl PyCustomClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096 ))] #[allow(clippy::too_many_arguments)] @@ -179,6 +183,7 @@ impl PyCustomClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, ) -> PyResult { // Convert the Python schema into serde JSON and pair it with the target-selector policy; @@ -191,6 +196,7 @@ impl PyCustomClassifierConfig { inner.classify_trigger = classify_trigger(session_affinity); inner.message_hash_fallback = message_hash_fallback; inner.recent_turn_window = recent_turn_window; + inner.judge_char_budget = judge_char_budget; inner.max_output_tokens = max_output_tokens; Ok(Self { inner }) } @@ -262,6 +268,7 @@ impl PyTaskClassifierConfig { session_affinity=false, message_hash_fallback=false, recent_turn_window=None, + judge_char_budget=18_000, max_output_tokens=4096, prompt=None, response_format_type="json_schema" @@ -273,6 +280,7 @@ impl PyTaskClassifierConfig { session_affinity: bool, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, prompt: Option, response_format_type: &str, @@ -284,6 +292,7 @@ impl PyTaskClassifierConfig { classify_trigger: classify_trigger(session_affinity), message_hash_fallback, recent_turn_window, + judge_char_budget, contract: classifier_contract(prompt, response_format_type)?, max_output_tokens, }, diff --git a/crates/switchyard-runner/src/algorithm.rs b/crates/switchyard-runner/src/algorithm.rs index c0630f199..b3acc52cb 100644 --- a/crates/switchyard-runner/src/algorithm.rs +++ b/crates/switchyard-runner/src/algorithm.rs @@ -106,6 +106,7 @@ struct CapabilityClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, prompt: Option, response_format_type: ClassifierResponseFormat, max_output_tokens: u64, @@ -132,6 +133,7 @@ struct CustomClassifierRouteConfig { classify_trigger: ClassifyTrigger, message_hash_fallback: bool, recent_turn_window: Option, + judge_char_budget: usize, max_output_tokens: u64, } @@ -245,6 +247,11 @@ pub struct LlmClassifierRouteConfig { /// How many trailing turns the judge sees. Unset shows it the opening task /// and the latest user follow-up only. pub recent_turn_window: Option, + /// Most characters the judge payload may use, in every mode. A window narrows from + /// the oldest turn until it fits, so one large tool result cannot decide the judge's + /// cost; without a window the task messages are clipped instead. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. Required in custom mode. pub prompt: Option, /// How the judge is asked for structured output. Use `json_object` when the @@ -464,6 +471,9 @@ pub struct StageClassifierConfig { /// and the latest user follow-up only. #[serde(default)] pub recent_turn_window: Option, + /// Most characters the judge payload may use. + #[serde(default = "default_judge_char_budget")] + pub judge_char_budget: usize, /// Replaces the packaged judge prompt. #[serde(default)] pub prompt: Option, @@ -511,6 +521,7 @@ impl StageClassifierConfig { classify_trigger: self.classify_trigger, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, + judge_char_budget: self.judge_char_budget, contract: classifier_contract(self.prompt.as_deref()) .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, @@ -847,6 +858,7 @@ impl LlmClassifierRouteConfig { classify_trigger, message_hash_fallback, recent_turn_window, + judge_char_budget, prompt, response_format_type, max_output_tokens, @@ -902,6 +914,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, @@ -948,7 +961,10 @@ impl LlmClassifierRouteConfig { prompt: prompt.clone(), response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, - judge: required_classifier_field(route_name, "escalation", escalation)?, + judge: EscalationJudgeConfig { + judge_char_budget: *judge_char_budget, + ..required_classifier_field(route_name, "escalation", escalation)? + }, }, )) } @@ -997,6 +1013,7 @@ impl LlmClassifierRouteConfig { classify_trigger: *classify_trigger, message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, + judge_char_budget: *judge_char_budget, max_output_tokens: *max_output_tokens, }, )) @@ -1083,6 +1100,7 @@ fn build_subagent_router_config( config.policy.into_libsy(), ); classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; let classifier = Arc::new( LlmTaskClassifier::new(LlmClassifierConfig::Custom { @@ -1180,6 +1198,7 @@ fn build_algorithm( classify_trigger: config.classify_trigger, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, + judge_char_budget: config.judge_char_budget, contract: classifier_contract(config.prompt.as_deref()) .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, @@ -1216,6 +1235,7 @@ fn build_algorithm( classifier_config.classify_trigger = config.classify_trigger; classifier_config.message_hash_fallback = config.message_hash_fallback; classifier_config.recent_turn_window = config.recent_turn_window; + classifier_config.judge_char_budget = config.judge_char_budget; classifier_config.max_output_tokens = config.max_output_tokens; LlmTaskClassifier::new(LlmClassifierConfig::Custom { default_target: config.default_target, @@ -1438,6 +1458,10 @@ fn warn_single_target_classifier(route_name: &str, models: &CategoryModelConfig) } } +fn default_judge_char_budget() -> usize { + TaskClassifierConfig::default().judge_char_budget +} + fn resolve_target_model_id( route_name: &str, name: &str, diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index b6f9ca58d..97c24321c 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1205,6 +1205,34 @@ new = ["send_message"] Ok(()) } + /// The route-level budget reaches every judge: a value the escalation judge rejects + /// fails the build, so it is not silently left at the default there. + #[test] + fn classifier_judge_char_budget_applies_in_every_mode() -> RunnerResult<()> { + let capability = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 1000", + ); + runner_from_toml(&capability)?; + + let escalation = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 1000\nescalation = { confirmations = 2 }", + ); + runner_from_toml(&escalation)?; + + let too_small = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\njudge_char_budget = 100\nescalation = { confirmations = 2 }", + ); + assert!( + error_message(&too_small).contains("judge_char_budget must be at least 256"), + "{}", + error_message(&too_small) + ); + Ok(()) + } + #[test] fn classifier_prompts_are_configurable_in_both_modes() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( @@ -1295,6 +1323,12 @@ efficient_target = "weak" ); assert!(error_message(&nested_completion_cap).contains("unknown field")); + let nested_char_budget = VALID_CONFIG.replace( + "base_threshold = 0.5", + "base_threshold = 0.5\nescalation = { judge_char_budget = 1000 }", + ); + assert!(error_message(&nested_char_budget).contains("unknown field")); + let unknown_classifier_field = VALID_CONFIG.replace( "base_threshold = 0.5", "base_threshold = 0.5\nclassifier_magic = true", diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 4be77ed86..fe978a2d9 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -209,13 +209,14 @@ checkpoint = "/models/router.pt" ### `llm_classifier` Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. -`max_output_tokens` applies to all three. +`max_output_tokens` and `judge_char_budget` apply to all three. | Key | Required | Default | Meaning | |---|:---:|---|---| | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Capability, escalation | — | Target the judge is called through. Not a routing destination. Custom mode uses `models.judge`. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | +| `judge_char_budget` | No | `18000` | Most characters sent to the judge per call, so one large tool result cannot decide judge cost and latency. A window narrows from the oldest turn until it fits; without a window the task messages are clipped, marked with `...[trimmed]`. Roughly four characters per token. Must be at least `256`. | | `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | Capability mode classifies before serving. See diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index 3fecddab6..ae181119c 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -110,6 +110,10 @@ configuration, so a bare `escalation = {}` is a valid, tuned route: | `recent_turn_window` | `28` | Trailing messages shown to the judge on top of the anchors. Must be at least `1`. | | `window_message_chars` | `500` | Per-message truncation cap inside that trailing window. Must be at least `50`. | +The route-level `judge_char_budget` (default `18000`) caps the whole transcript +sent to the judge. The per-message caps normally keep it well under that; when +they do not, the oldest window lines are dropped first. + `confirmations` is the main cost dial. `1` latches sooner and spends more on the strong tier. `2` or higher requires a session identity, because the streak is retained per session — without one, every turn starts from zero and the route diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index c1b1af8af..58e957d6f 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -122,6 +122,7 @@ for the server merge behavior. | `base_threshold` | required | Lowest `p_solve` that routes a supported task to `weak_target`. Must be between `0` and `1`. | | `threshold_step` | `0.0` | Amount added for each boundary step. Must be finite and non-negative, and `base_threshold + 2 * threshold_step` must not exceed `1`. | | `recent_turn_window` | unset | When unset, the judge sees the opening user task and the latest user message when they differ. When set to `N`, it sees the opening user task and the last `N` conversation messages after that task. `0` keeps only the opening task. Client system and developer instructions are not shown to the judge. | +| `judge_char_budget` | `18000` | Most characters sent to the judge per call, roughly four per token. A window is counted in turns, and one turn carrying a large tool result can be worth tens of thousands of characters, so the window ends at the first message that does not fit. A task statement larger than the whole budget is clipped instead, marked with `...[trimmed]`. Must be at least `256`. | | `classify_trigger` | `every_request` | When the judge runs. `every_request` judges every request, tool continuations included. `user_turn` judges each new user message and holds that target across the tool calls between. `new_session` judges once and reuses that target for the session. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `classify_trigger = "new_session"`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index fd768ba71..6065882c1 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -63,7 +63,8 @@ class CustomClassifierConfig: """Configure schema-validated routing across runtime model groups. ``max_output_tokens`` must be positive. Enabling ``message_hash_fallback`` - requires ``session_affinity``. + requires ``session_affinity``. ``judge_char_budget`` caps the characters sent + to the judge and must be at least 256. """ def __init__( @@ -75,6 +76,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, ) -> None: ... @@ -82,8 +84,9 @@ def __init__( class EscalationClassifierConfig: """Configure response-based escalation between two targets. - Counts and token limits must be positive, and ``window_message_chars`` - must be at least 50. + Counts and token limits must be positive, ``window_message_chars`` must be + at least 50, and ``judge_char_budget`` caps the characters sent to the judge + and must be at least 256. """ def __init__( @@ -92,6 +95,7 @@ def __init__( confirmations: int = 2, recent_turn_window: int = 28, window_message_chars: int = 500, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema", @@ -156,6 +160,8 @@ class TaskClassifierConfig: Thresholds must remain within ``[0, 1]``, ``max_output_tokens`` must be positive, and ``message_hash_fallback`` requires ``session_affinity``. + ``judge_char_budget`` caps the characters sent to the judge and must be at + least 256. """ def __init__( @@ -166,6 +172,7 @@ def __init__( session_affinity: bool = False, message_hash_fallback: bool = False, recent_turn_window: int | None = None, + judge_char_budget: int = 18_000, max_output_tokens: int = 4096, prompt: str | None = None, response_format_type: Literal["json_schema", "json_object"] = "json_schema", From 700fabfcae0da4926764d334eaf8ffb8fb91e50f Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Fri, 18 Sep 2026 19:28:48 -0700 Subject: [PATCH 2/2] test(runner): cover judge_char_budget forwarding in custom mode Signed-off-by: Arnav Dadarya --- crates/switchyard-runner/src/config.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/switchyard-runner/src/config.rs b/crates/switchyard-runner/src/config.rs index 97c24321c..03ded893e 100644 --- a/crates/switchyard-runner/src/config.rs +++ b/crates/switchyard-runner/src/config.rs @@ -1205,8 +1205,8 @@ new = ["send_message"] Ok(()) } - /// The route-level budget reaches every judge: a value the escalation judge rejects - /// fails the build, so it is not silently left at the default there. + /// The route-level budget reaches every judge: a value a judge rejects fails the + /// build, so it is not silently left at the default in any mode. #[test] fn classifier_judge_char_budget_applies_in_every_mode() -> RunnerResult<()> { let capability = VALID_CONFIG.replace( @@ -1230,6 +1230,18 @@ new = ["send_message"] "{}", error_message(&too_small) ); + + let custom = + with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "\njudge_char_budget = 1000"); + runner_from_toml(&custom)?; + + let custom_too_small = + with_subagent_llm_classifier(VALID_CONFIG, "passthrough", "\njudge_char_budget = 100"); + assert!( + error_message(&custom_too_small).contains("judge_char_budget must be at least 256"), + "{}", + error_message(&custom_too_small) + ); Ok(()) }