From 389950a36ab61fa6f9b0457d8db9f5be88ae8a90 Mon Sep 17 00:00:00 2001 From: uiharuayako <2451602772@qq.com> Date: Tue, 27 Oct 2026 12:51:17 +0800 Subject: [PATCH 1/2] fix: enforce configurable per-model image input limits --- .../src-tauri/src/installer/ai_config.rs | 1 + src/apps/cli/src/model_selection.rs | 3 + src/apps/desktop/README.md | 18 + src/crates/adapters/ai-adapters/AGENTS.md | 6 + src/crates/adapters/ai-adapters/src/client.rs | 49 +- .../ai-adapters/src/client/image_budget.rs | 594 ++++++++++++++++++ .../adapters/ai-adapters/src/client/sse.rs | 1 + .../ai-adapters/src/providers/openai/chat.rs | 1 + .../src/providers/openai/responses.rs | 1 + .../src/subscription_auth/request_identity.rs | 1 + .../src/agentic/coordination/coordinator.rs | 317 ++++++++++ .../src/agentic/coordination/scheduler.rs | 164 ++++- .../core/src/agentic/execution/AGENTS.md | 10 + .../src/agentic/execution/execution_engine.rs | 130 ++++ .../agentic/execution/prompt_suggestion.rs | 1 + .../src/agentic/execution/round_executor.rs | 1 + .../core/src/agentic/image_generation/mod.rs | 1 + .../core/src/agentic/memories/service.rs | 1 + .../core/src/function_agents/port_adapters.rs | 1 + .../src/infrastructure/ai/client_factory.rs | 1 + .../infrastructure/ai/reasoning_catalog.rs | 1 + .../core/src/service/config/providers.rs | 21 + .../src/service/token_usage/statistics.rs | 1 + .../config-contracts/src/conversion.rs | 99 +++ .../config-contracts/src/normalization.rs | 34 + .../contracts/config-contracts/src/types.rs | 5 + src/crates/contracts/core-types/src/ai.rs | 3 + .../app-server/src/management/owner.rs | 4 + .../ai/ModelSettingsPage.loading.test.tsx | 36 ++ .../settings/pages/ai/ModelSettingsPage.tsx | 24 + .../src/flow_chat/components/ChatInput.tsx | 52 +- .../components/ChatInputImageIntake.test.ts | 14 + .../flow_chat/components/ModelSelector.tsx | 9 +- .../ModelSelectorProviderLevels.test.tsx | 27 + .../constants/chatInputConfig.test.ts | 16 + .../flow_chat/constants/chatInputConfig.ts | 7 + .../flow_chat/utils/modelImageBudget.test.ts | 43 ++ .../src/flow_chat/utils/modelImageBudget.ts | 19 + .../modelEditorRequestSettings.test.ts | 14 + .../components/modelEditorRequestSettings.ts | 2 + .../src/infrastructure/config/types/index.ts | 2 + .../src/locales/en-US/settings/models.json | 2 + .../src/locales/zh-CN/settings/models.json | 2 + .../src/locales/zh-TW/settings/models.json | 2 + 44 files changed, 1684 insertions(+), 57 deletions(-) create mode 100644 src/crates/adapters/ai-adapters/src/client/image_budget.rs create mode 100644 src/web-ui/src/flow_chat/constants/chatInputConfig.test.ts create mode 100644 src/web-ui/src/flow_chat/utils/modelImageBudget.test.ts create mode 100644 src/web-ui/src/flow_chat/utils/modelImageBudget.ts diff --git a/OpenBitFun-Installer/src-tauri/src/installer/ai_config.rs b/OpenBitFun-Installer/src-tauri/src/installer/ai_config.rs index 2ac32d8273..61e455e8cc 100644 --- a/OpenBitFun-Installer/src-tauri/src/installer/ai_config.rs +++ b/OpenBitFun-Installer/src-tauri/src/installer/ai_config.rs @@ -45,6 +45,7 @@ pub(super) fn ai_config_from_installer_model(m: &ModelConfig) -> Result, ) -> Result { let custom_body = self.config.custom_request_body.clone(); - if matches!( - ApiFormat::parse(&self.config.format)?, - ApiFormat::OpenAIResponses - ) { - return openai::responses::send_stream( - self, - messages, - tools, - custom_body, - 1, - trace, - request_context, - ) - .await; - } - self.send_message_stream_with_extra_body_and_max_attempts( messages, tools, @@ -611,6 +596,10 @@ impl AIClient { trace: Option, request_context: Option, ) -> Result { + // Project only the outgoing copy, against the actual selected client. + // Account/channel failover and direct analysis calls share this boundary. + let messages = image_budget::project(messages, self.config.max_images_per_request)?; + image_budget::validate_extra_body(self.config.max_images_per_request, extra_body.as_ref())?; match ApiFormat::parse(&self.config.format)? { ApiFormat::OpenAIChat => { openai::chat::send_stream( @@ -871,6 +860,7 @@ mod tests { format: format.to_string(), context_window: 128000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1335,6 +1325,7 @@ mod tests { format: "openai".to_string(), context_window: 128000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1368,6 +1359,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1400,6 +1392,7 @@ mod tests { format: "gemini".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: Some(0.2), top_p: Some(0.8), inline_think_in_text: false, @@ -1482,6 +1475,7 @@ mod tests { format: "gemini".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1529,6 +1523,7 @@ mod tests { format: "openai".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1566,6 +1561,7 @@ mod tests { format: "openai".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1605,6 +1601,7 @@ mod tests { format: "openai".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1642,6 +1639,7 @@ mod tests { format: "openai".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1683,6 +1681,7 @@ mod tests { format: "openai".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1860,6 +1859,7 @@ mod tests { format: "openai".to_string(), context_window: 128_000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1899,6 +1899,7 @@ mod tests { format: "openai".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1936,6 +1937,7 @@ mod tests { format: "responses".to_string(), context_window: 128000, max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -1977,6 +1979,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2017,6 +2020,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2056,6 +2060,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2095,6 +2100,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2136,6 +2142,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2170,6 +2177,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2211,6 +2219,7 @@ mod tests { format: "anthropic".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2253,6 +2262,7 @@ mod tests { format: "anthropic".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2293,6 +2303,7 @@ mod tests { format: "anthropic".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2331,6 +2342,7 @@ mod tests { format: "anthropic".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2371,6 +2383,7 @@ mod tests { format: "anthropic".to_string(), context_window: 1_000_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2413,6 +2426,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(4000), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2451,6 +2465,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2485,6 +2500,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, @@ -2523,6 +2539,7 @@ mod tests { format: "anthropic".to_string(), context_window: 200000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/adapters/ai-adapters/src/client/image_budget.rs b/src/crates/adapters/ai-adapters/src/client/image_budget.rs new file mode 100644 index 0000000000..c9eeb8aa4a --- /dev/null +++ b/src/crates/adapters/ai-adapters/src/client/image_budget.rs @@ -0,0 +1,594 @@ +//! Model-specific image input budgets for an outgoing request copy. + +use crate::types::Message; +use anyhow::Result; +use openbitfun_core_types::errors::{AiProviderError, ErrorCategory}; +use serde_json::{json, Value}; + +/// Message payload overrides are merged after the message projection. Reject +/// them when a budget is configured rather than accepting an uncounted payload. +pub(super) fn validate_extra_body( + max_images: Option, + extra_body: Option<&Value>, +) -> Result<()> { + if max_images.is_some() + && extra_body.and_then(Value::as_object).is_some_and(|fields| { + ["messages", "input", "contents"] + .iter() + .any(|key| fields.contains_key(*key)) + }) + { + return Err(AiProviderError::classified( + "A model image limit cannot be combined with custom messages, input or contents. Remove the message payload override and use ordinary model messages.".into(), + ErrorCategory::InvalidRequest, + ) + .into()); + } + Ok(()) +} + +/// Keep the newest image-bearing messages within the selected model's budget. +/// Within one message, retain the first images in their original order. Tool +/// identities, text, reasoning and replay state are otherwise left intact. +pub(super) fn project( + mut messages: Vec, + max_images: Option, +) -> Result> { + let Some(mut remaining) = max_images else { + return Ok(messages); + }; + if remaining == 0 { + return Err(AiProviderError::classified( + "max_images_per_request must be greater than zero".into(), + ErrorCategory::InvalidRequest, + ) + .into()); + } + + for message in messages.iter_mut().rev() { + match message.role.as_str() { + "tool" => { + let Some(images) = message.tool_image_attachments.as_mut() else { + continue; + }; + let total = images.len(); + let keep = total.min(remaining); + remaining -= keep; + let omitted = total - keep; + if omitted > 0 { + images.truncate(keep); + if images.is_empty() { + message.tool_image_attachments = None; + } + let text = message.content.get_or_insert_with(String::new); + let retained = if keep > 0 { + format!(" Only the first {keep} of {total} images from this tool result are included.") + } else { + String::new() + }; + text.push_str(&format!( + "\n\n[{omitted} tool image(s) omitted to fit the model's per-request image limit.{retained} Read images in smaller batches if their pixels are needed.]" + )); + } + } + "user" => project_user_content(&mut message.content, &mut remaining), + _ => {} + } + } + Ok(messages) +} + +fn project_user_content(content: &mut Option, remaining: &mut usize) { + let Some(original) = content.as_deref() else { + return; + }; + let Ok(Value::Array(mut parts)) = serde_json::from_str::(original) else { + return; + }; + // Ordinary JSON remains text. Require a recognizable content-part array, + // rather than recursively treating arbitrary image-looking fields as pixels. + if !parts.iter().all(is_content_part) { + return; + } + let image_count = parts.iter().filter(|part| is_image_part(part)).count(); + let keep = image_count.min(*remaining); + *remaining -= keep; + let omitted = image_count - keep; + if omitted == 0 { + return; + } + let mut seen = 0usize; + parts.retain(|part| { + if is_image_part(part) { + seen += 1; + seen <= keep + } else { + true + } + }); + let retained = if keep > 0 { + format!(" Only the first {keep} of {image_count} images from this message are included.") + } else { + String::new() + }; + parts.push(json!({ + "type": "text", + "text": format!("[{omitted} image(s) omitted to fit the model's per-request image limit.{retained} Reattach or read images in smaller batches if their pixels are needed.]"), + })); + *content = Some(Value::Array(parts).to_string()); +} + +fn image_url(value: Option<&Value>) -> bool { + value + .and_then(|value| value.as_str().or_else(|| value.get("url")?.as_str())) + .is_some_and(|url| !url.is_empty()) +} + +fn inline_media(part: &Value) -> Option<(&str, &str)> { + let data = part.get("inline_data").or_else(|| part.get("inlineData"))?; + Some(( + data.get("mime_type") + .or_else(|| data.get("mimeType"))? + .as_str()?, + data.get("data")?.as_str()?, + )) +} + +fn is_image_part(part: &Value) -> bool { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => image_url(part.get("image_url")), + Some("input_image") => { + image_url(part.get("image_url")) + || part.get("file_id").and_then(Value::as_str).is_some() + } + Some("image") => part.get("source").is_some_and(|source| { + image_url(source.get("url")) + || (source + .get("media_type") + .and_then(Value::as_str) + .is_some_and(|mime| mime.starts_with("image/")) + && source.get("data").and_then(Value::as_str).is_some()) + }), + _ => inline_media(part).is_some_and(|(mime, _)| mime.starts_with("image/")), + } +} + +fn is_content_part(part: &Value) -> bool { + if is_image_part(part) || inline_media(part).is_some() { + return true; + } + match part.get("type").and_then(Value::as_str) { + Some("text" | "input_text" | "output_text") | None => { + part.get("text").and_then(Value::as_str).is_some() + } + // Preserve non-image multimodal parts without charging the image budget. + Some("input_audio" | "audio" | "video" | "file" | "input_file" | "document") => true, + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::providers::anthropic::message_converter::AnthropicMessageConverter; + use crate::providers::gemini::message_converter::GeminiMessageConverter; + use crate::providers::openai::message_converter::OpenAIMessageConverter; + use crate::types::ToolImageAttachment; + + fn image(id: &str) -> Value { + json!({"type": "image_url", "image_url": {"url": format!("data:image/png;base64,{id}")}}) + } + + fn user_images(ids: &[&str]) -> Message { + let mut parts = vec![json!({"type": "text", "text": "Inspect these images."})]; + parts.extend(ids.iter().map(|id| image(id))); + Message::user(Value::Array(parts).to_string()) + } + + fn tool_images(id: &str, count: usize) -> Message { + let mut message = Message::user("Image attached".into()); + message.role = "tool".into(); + message.tool_call_id = Some(id.into()); + message.name = Some("view_image".into()); + message.tool_image_attachments = Some( + (0..count) + .map(|index| ToolImageAttachment { + mime_type: "image/png".into(), + data_base64: format!("{id}_{index}"), + }) + .collect(), + ); + message + } + + fn count(messages: &[Message]) -> usize { + messages + .iter() + .map(|message| { + let tool_count = message.tool_image_attachments.as_ref().map_or(0, Vec::len); + let user_count = if message.role == "user" { + message + .content + .as_deref() + .and_then(|content| serde_json::from_str::(content).ok()) + .and_then(|value| value.as_array().cloned()) + .map_or(0, |parts| { + parts.iter().filter(|part| is_image_part(part)).count() + }) + } else { + 0 + }; + tool_count + user_count + }) + .sum() + } + + fn wire_count(value: &Value) -> usize { + match value { + Value::Array(items) => items.iter().map(wire_count).sum(), + Value::Object(fields) => { + usize::from(is_image_part(value)) + fields.values().map(wire_count).sum::() + } + _ => 0, + } + } + + fn client_config(format: &str, url: &str, limit: usize) -> crate::types::AIConfig { + serde_json::from_value(json!({ + "name": "image-budget-test", + "base_url": url, + "request_url": url, + "api_key": "synthetic", + "model": "test", + "format": format, + "context_window": 4096, + "max_images_per_request": limit, + "inline_think_in_text": false, + "skip_ssl_verify": false + })) + .unwrap() + } + + #[test] + fn mixed_user_and_tool_images_share_the_request_budget() { + let original = vec![user_images(&["old-a", "old-b"]), tool_images("latest", 2)]; + let original_json = serde_json::to_value(&original).unwrap(); + let projected = project(original.clone(), Some(3)).unwrap(); + assert_eq!(count(&projected), 3); + let content = projected[0].content.as_deref().unwrap(); + assert!(content.contains("old-a")); + assert!(!content.contains("old-b")); + assert!(content.contains("1 image(s) omitted")); + assert!(content.contains("first 1 of 2 images")); + assert_eq!(projected[1].tool_call_id.as_deref(), Some("latest")); + assert_eq!(serde_json::to_value(original).unwrap(), original_json); + } + + #[test] + fn four_parallel_tool_results_keep_the_latest_three_without_losing_results() { + let messages = (0..4) + .map(|index| tool_images(&format!("call-{index}"), 1)) + .collect(); + let projected = project(messages, Some(3)).unwrap(); + assert_eq!(projected.len(), 4); + assert_eq!(count(&projected), 3); + assert!(projected[0].tool_image_attachments.is_none()); + assert!(projected[0] + .content + .as_deref() + .unwrap() + .contains("Read images in smaller batches")); + for (index, message) in projected.iter().enumerate() { + assert_eq!(message.tool_call_id, Some(format!("call-{index}"))); + assert_eq!(message.name.as_deref(), Some("view_image")); + assert!(message + .content + .as_deref() + .unwrap() + .starts_with("Image attached")); + } + } + + #[test] + fn one_tool_result_can_be_partially_projected() { + let mut original = tool_images("batch", 4); + original.reasoning_content = Some("keep reasoning".into()); + original.thinking_signature = Some("keep signature".into()); + original.is_error = Some(false); + original.model_response_replay = Some( + serde_json::from_value(json!({ + "protocol": "test", "items": [] + })) + .unwrap(), + ); + let mut expected = serde_json::to_value(&original).unwrap(); + let projected = project(vec![original], Some(3)).unwrap(); + let images = projected[0].tool_image_attachments.as_ref().unwrap(); + assert_eq!(images.len(), 3); + assert_eq!(images[0].data_base64, "batch_0"); + assert_eq!(images[2].data_base64, "batch_2"); + assert!(projected[0] + .content + .as_deref() + .unwrap() + .contains("1 tool image(s) omitted")); + assert!(projected[0] + .content + .as_deref() + .unwrap() + .contains("first 3 of 4 images")); + // The content placeholder and truncated attachment vector are the only + // permitted differences; metadata and result identities remain intact. + let actual = serde_json::to_value(&projected[0]).unwrap(); + expected["content"] = actual["content"].clone(); + expected["tool_image_attachments"] = actual["tool_image_attachments"].clone(); + assert_eq!(actual, expected); + } + + #[test] + fn a_plain_continue_reprojects_old_history_with_the_actual_clients_limit() { + let mut history = vec![tool_images("batch", 4)]; + assert_eq!(count(&project(history.clone(), Some(5)).unwrap()), 4); + assert_eq!(count(&project(history.clone(), Some(3)).unwrap()), 3); + history.push(Message::user("Continue.".into())); + let projected = project(history.clone(), Some(3)).unwrap(); + assert_eq!(count(&projected), 3); + assert_eq!(projected[1].content.as_deref(), Some("Continue.")); + assert_eq!(count(&history), 4); + assert_eq!(count(&project(projected, Some(3)).unwrap()), 3); + } + + #[test] + fn unconfigured_and_within_budget_requests_are_byte_for_byte_unchanged() { + let mut message = user_images(&["a", "b", "c", "d"]); + message.content = Some(format!("\n {} \n", message.content.unwrap())); + message.reasoning_content = Some("Reasoning remains attached.".into()); + message.thinking_signature = Some("signature".into()); + let messages = vec![message, tool_images("tool", 1)]; + let expected = serde_json::to_value(&messages).unwrap(); + assert_eq!( + serde_json::to_value(project(messages.clone(), None).unwrap()).unwrap(), + expected + ); + assert_eq!( + serde_json::to_value(project(messages, Some(5)).unwrap()).unwrap(), + expected + ); + } + + #[test] + fn zero_limit_is_a_structured_invalid_request() { + let error = project(vec![Message::user("hello".into())], Some(0)).unwrap_err(); + let error = error.downcast_ref::().unwrap(); + assert_eq!(error.category, ErrorCategory::InvalidRequest); + assert!(error.provider.is_none()); + assert!(error.http_status.is_none()); + } + + #[test] + fn message_payload_overrides_cannot_bypass_a_configured_image_limit() { + for key in ["messages", "input", "contents"] { + let override_body = json!({(key): [image("custom")]}); + let error = validate_extra_body(Some(3), Some(&override_body)).unwrap_err(); + let error = error.downcast_ref::().unwrap(); + assert_eq!(error.category, ErrorCategory::InvalidRequest); + assert!(error.provider.is_none()); + assert!(error.http_status.is_none()); + assert!(error + .message + .contains("Remove the message payload override")); + assert!(validate_extra_body(None, Some(&override_body)).is_ok()); + } + assert!(validate_extra_body(Some(3), None).is_ok()); + assert!(validate_extra_body( + Some(3), + Some(&json!({ + "temperature": 0.2, "thinking": {"type": "enabled"} + })) + ) + .is_ok()); + } + + #[tokio::test] + async fn every_send_entry_point_rejects_a_zero_budget_before_transport() { + for format in ["openai", "responses", "anthropic", "gemini"] { + let config = client_config(format, "invalid-image-budget-test", 0); + let client = crate::AIClient::new(config); + let messages = vec![Message::user("hello".into())]; + let errors = [ + client + .send_message_stream_once(messages.clone(), None, None) + .await + .err() + .unwrap(), + client + .send_message_once_with_trace_and_request_context( + messages.clone(), + None, + None, + None, + ) + .await + .err() + .unwrap(), + ]; + for error in errors { + assert_eq!( + error.downcast_ref::().unwrap().category, + ErrorCategory::InvalidRequest + ); + } + for key in ["messages", "input", "contents"] { + let mut bounded = client.clone(); + bounded.config.max_images_per_request = Some(3); + bounded.config.custom_request_body = Some(json!({(key): []})); + let error = bounded + .send_message_stream_once(messages.clone(), None, None) + .await + .err() + .unwrap(); + assert_eq!( + error.downcast_ref::().unwrap().category, + ErrorCategory::InvalidRequest + ); + } + } + } + + #[tokio::test] + async fn selected_client_budget_is_applied_to_each_actual_http_send() { + use axum::{extract::State, routing::post, Json, Router}; + use std::sync::{Arc, Mutex}; + + async fn receive( + State(requests): State>>>, + Json(body): Json, + ) -> ([(&'static str, &'static str); 1], String) { + requests.lock().unwrap().push(body); + let content_chunk = json!({ + "id": "image-budget-test", + "object": "chat.completion.chunk", + "created": 1, + "model": "test", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": "ok"}, + "finish_reason": null + }] + }); + let stop_chunk = json!({ + "id": "image-budget-test", + "object": "chat.completion.chunk", + "created": 1, + "model": "test", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}] + }); + ( + [("content-type", "text/event-stream")], + format!("data: {content_chunk}\n\ndata: {stop_chunk}\n\ndata: [DONE]\n\n"), + ) + } + + let requests = Arc::new(Mutex::new(Vec::::new())); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/chat/completions", listener.local_addr().unwrap()); + let router = Router::new() + .route("/chat/completions", post(receive)) + .with_state(requests.clone()); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let initial = crate::AIClient::new(client_config("openai", &url, 5)); + let mut selected = initial.clone(); + selected.config.max_images_per_request = Some(3); + let mut history = vec![tool_images("batch", 4)]; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + initial + .send_message_once_with_trace_and_request_context(history.clone(), None, None, None) + .await + .unwrap(); + selected + .send_message_once_with_trace_and_request_context(history.clone(), None, None, None) + .await + .unwrap(); + history.push(Message::user("Continue.".into())); + selected + .send_message_once_with_trace_and_request_context(history.clone(), None, None, None) + .await + .unwrap(); + }) + .await + .unwrap(); + server.abort(); + let requests = requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert_eq!(wire_count(&requests[0]["messages"]), 4); + assert_eq!(wire_count(&requests[1]["messages"]), 3); + assert_eq!(wire_count(&requests[2]["messages"]), 3); + assert_eq!(count(&history), 4); + } + + #[test] + fn tool_json_is_opaque_even_if_it_looks_like_image_content() { + let mut message = tool_images("opaque", 0); + message.content = user_images(&["a", "b", "c", "d"]).content; + let expected = serde_json::to_value(&message).unwrap(); + assert_eq!( + serde_json::to_value(&project(vec![message], Some(1)).unwrap()[0]).unwrap(), + expected + ); + } + + #[test] + fn ordinary_json_and_non_image_inline_media_do_not_consume_the_budget() { + let ordinary = Message::user(json!({"images": [image("metadata")]}).to_string()); + let array = Message::user(json!([{"record": image("metadata")}]).to_string()); + let audio = Message::user( + json!([ + {"inlineData": {"mimeType": "audio/wav", "data": "sound"}}, + {"type": "text", "text": "transcribe"} + ]) + .to_string(), + ); + let expected = serde_json::to_value([&ordinary, &array, &audio]).unwrap(); + let projected = project( + vec![ordinary, array, audio, tool_images("latest", 1)], + Some(1), + ) + .unwrap(); + assert_eq!(serde_json::to_value(&projected[..3]).unwrap(), expected); + assert_eq!(count(&projected), 1); + } + + #[test] + fn recognizes_all_image_content_dialects_and_keeps_non_image_parts() { + let dialects = [ + image("pixels"), + json!({"type": "input_image", "image_url": "data:image/png;base64,pixels"}), + json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "pixels"}}), + json!({"inline_data": {"mime_type": "image/png", "data": "pixels"}}), + json!({"inlineData": {"mimeType": "image/png", "data": "pixels"}}), + ]; + for image_part in dialects { + let text = json!({"type": "text", "text": "Preserve this text."}); + let parts = vec![ + image_part.clone(), + text.clone(), + image_part.clone(), + image_part, + ]; + let projected = + project(vec![Message::user(json!(parts).to_string())], Some(2)).unwrap(); + let parts: Vec = + serde_json::from_str(projected[0].content.as_deref().unwrap()).unwrap(); + assert_eq!(parts.iter().filter(|part| is_image_part(part)).count(), 2); + assert_eq!(parts[1], text); + assert!(parts.last().unwrap()["text"] + .as_str() + .unwrap() + .contains("1 image(s) omitted")); + } + } + + #[test] + fn projected_pixels_stay_within_budget_in_every_provider_wire_format() { + for format in ["openai", "responses", "anthropic", "gemini"] { + let mut user = user_images(&["a", "b"]); + if format == "anthropic" { + user.content = Some(json!([ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "a"}}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "b"}}, + {"type": "text", "text": "Inspect these images."} + ]).to_string()); + } + let projected = project(vec![user, tool_images("a", 2)], Some(3)).unwrap(); + let wire = match format { + "openai" => json!(OpenAIMessageConverter::convert_messages(projected)), + "responses" => { + json!(OpenAIMessageConverter::convert_messages_to_responses_input(projected).1) + } + "anthropic" => json!(AnthropicMessageConverter::convert_messages(projected).1), + _ => json!(GeminiMessageConverter::convert_messages(projected, "gemini-3-pro").1), + }; + assert_eq!(wire_count(&wire), 3, "format={format}, wire={wire}"); + } + } +} diff --git a/src/crates/adapters/ai-adapters/src/client/sse.rs b/src/crates/adapters/ai-adapters/src/client/sse.rs index 305c1396c5..ab120adedd 100644 --- a/src/crates/adapters/ai-adapters/src/client/sse.rs +++ b/src/crates/adapters/ai-adapters/src/client/sse.rs @@ -1492,6 +1492,7 @@ mod tests { format: "openai".into(), context_window: 4096, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/adapters/ai-adapters/src/providers/openai/chat.rs b/src/crates/adapters/ai-adapters/src/providers/openai/chat.rs index a9a169b49c..b218bcc946 100644 --- a/src/crates/adapters/ai-adapters/src/providers/openai/chat.rs +++ b/src/crates/adapters/ai-adapters/src/providers/openai/chat.rs @@ -456,6 +456,7 @@ mod tests { format: "openai".into(), context_window: 4096, max_tokens: Some(32), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/adapters/ai-adapters/src/providers/openai/responses.rs b/src/crates/adapters/ai-adapters/src/providers/openai/responses.rs index a984bb08c7..05bda1fe7d 100644 --- a/src/crates/adapters/ai-adapters/src/providers/openai/responses.rs +++ b/src/crates/adapters/ai-adapters/src/providers/openai/responses.rs @@ -386,6 +386,7 @@ mod tests { format: "responses".to_string(), context_window: 128_000, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/adapters/ai-adapters/src/subscription_auth/request_identity.rs b/src/crates/adapters/ai-adapters/src/subscription_auth/request_identity.rs index 9499aacff3..dc88f24976 100644 --- a/src/crates/adapters/ai-adapters/src/subscription_auth/request_identity.rs +++ b/src/crates/adapters/ai-adapters/src/subscription_auth/request_identity.rs @@ -224,6 +224,7 @@ mod tests { format: "responses".into(), context_window: 128_000, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/agentic/coordination/coordinator.rs b/src/crates/assembly/core/src/agentic/coordination/coordinator.rs index f587b4bf02..a6a01f686d 100644 --- a/src/crates/assembly/core/src/agentic/coordination/coordinator.rs +++ b/src/crates/assembly/core/src/agentic/coordination/coordinator.rs @@ -6279,6 +6279,17 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet .session_manager .get_session(session_id) .ok_or_else(|| OpenBitFunError::NotFound(format!("Session not found: {session_id}")))?; + // Queueing and live steering must reject an oversized user attachment + // batch while the sender still owns it. Do not guess unresolved defaults; + // a selected pool has a common limit independent of its current member. + if let Some(ai_config) = SessionManager::load_ai_config_for_model_resolution().await { + Self::validate_input_image_count( + &ai_config, + session.config.model_id.as_deref(), + Self::input_image_pool_id(&session.config), + images.len(), + )?; + } let workspace = Self::build_workspace_binding(&session.config).await; let context = crate::agentic::tools::framework::ToolUseContext::for_tool_listing(workspace, None); @@ -6288,6 +6299,83 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet .await } + fn input_image_pool_id(config: &SessionConfig) -> Option<&str> { + if config.model_binding_policy != SessionModelBindingPolicy::Pool { + return None; + } + config.model_pool_id.as_deref().or_else(|| { + config + .model_pool_binding + .as_ref() + .map(|binding| binding.pool_id.as_str()) + }) + } + + fn validate_input_image_count( + ai_config: &AIConfig, + resolved_model_id: Option<&str>, + pool_id: Option<&str>, + image_count: usize, + ) -> OpenBitFunResult<()> { + let roots: Vec<_> = if let Some(pool_id) = pool_id { + let pool = ai_config + .model_pools + .get(pool_id) + .map_err(OpenBitFunError::validation)?; + ai_config + .models + .iter() + .filter(|model| pool.member_model_ids.contains(&model.id)) + .collect() + } else { + ai_config + .models + .iter() + .filter(|model| Some(model.id.as_str()) == resolved_model_id) + .collect() + }; + // The complete new user batch must fit every native-image destination + // that the turn may use. Health and cooldown never widen this budget. + let mut limiting_model = None; + for root in roots + .into_iter() + .filter(|model| model.enabled && model.supports_text_generation()) + { + let route = root.channel_routing.as_ref(); + for candidate in ai_config.models.iter().filter(|candidate| { + candidate.enabled + && candidate.supports_text_generation() + && candidate.supports_image_understanding() + }) { + let selected = match route.and_then(|route| route.manual_channel.as_deref()) { + Some(manual) => candidate.id == manual, + None => { + candidate.id == root.id + || route.is_some_and(|route| { + route.fallback_model_ids.contains(&candidate.id) + }) + } + }; + if selected { + if let Some(limit) = candidate.max_images_per_request { + if limiting_model.is_none_or(|(_, current)| limit < current) { + limiting_model = Some((candidate, limit)); + } + } + } + } + } + if let Some((model, limit)) = limiting_model { + if image_count > limit { + return Err(OpenBitFunError::validation(format!( + "Too many images attached for model '{}': {} > {}. Send no more than {} images at a time.", + model.name, image_count, limit, limit + ))); + } + } + Ok(()) + } + #[allow(clippy::too_many_arguments)] async fn start_dialog_turn_internal( &self, @@ -6802,6 +6890,12 @@ Update the persona files and delete BOOTSTRAP.md as soon as bootstrap is complet let ai_config: crate::service::config::types::AIConfig = config_service.get_config(Some("ai")).await?; image_requirement.validate(&ai_config, &resolved_model_id)?; + Self::validate_input_image_count( + &ai_config, + Some(&resolved_model_id), + Self::input_image_pool_id(&session.config), + image_contexts.as_ref().map_or(0, Vec::len), + )?; } let reasoning_preset = self .session_manager @@ -16328,6 +16422,229 @@ pub(crate) mod tests { use openbitfun_services_core::permission_store::ProjectPermissionSqliteStore; use std::path::Path; + #[test] + fn input_image_count_admission_ignores_stale_pool_pointers_for_fixed_models() { + let mut config = SessionConfig { + model_id: Some("fixed-vision".into()), + model_pool_id: Some("stale-pool".into()), + model_pool_binding: Some(openbitfun_core_types::model_pool::ModelPoolBinding { + pool_id: "bound-pool".into(), + model_id: "previous-member".into(), + configuration_fingerprint: String::new(), + generation: 0, + }), + ..Default::default() + }; + for policy in [ + SessionModelBindingPolicy::Mutable, + SessionModelBindingPolicy::ApprovedImmutable, + ] { + config.model_binding_policy = policy; + assert_eq!(ConversationCoordinator::input_image_pool_id(&config), None); + } + config.model_binding_policy = SessionModelBindingPolicy::Pool; + assert_eq!( + ConversationCoordinator::input_image_pool_id(&config), + Some("stale-pool") + ); + config.model_pool_id = None; + assert_eq!( + ConversationCoordinator::input_image_pool_id(&config), + Some("bound-pool") + ); + } + + #[test] + fn input_image_count_admission_uses_the_resolved_native_model_limit() { + use crate::service::config::types::{AIConfig, AIModelConfig, ModelCapability}; + + let mut config = AIConfig { + models: vec![AIModelConfig { + id: "limited-vision".into(), + name: "Limited vision".into(), + enabled: true, + capabilities: vec![ + ModelCapability::TextChat, + ModelCapability::ImageUnderstanding, + ], + max_images_per_request: Some(3), + ..Default::default() + }], + ..Default::default() + }; + ConversationCoordinator::validate_input_image_count( + &config, + Some("limited-vision"), + None, + 3, + ) + .expect("the entire user batch fits"); + let error = ConversationCoordinator::validate_input_image_count( + &config, + Some("limited-vision"), + None, + 4, + ) + .expect_err("a user batch must not be silently truncated"); + assert!(matches!(error, OpenBitFunError::Validation(message) if message.contains("4 > 3"))); + + config.models[0].max_images_per_request = None; + ConversationCoordinator::validate_input_image_count( + &config, + Some("limited-vision"), + None, + 20, + ) + .expect("legacy models retain their existing behavior"); + config.models[0].max_images_per_request = Some(3); + config.models[0].capabilities = vec![ModelCapability::TextChat]; + ConversationCoordinator::validate_input_image_count( + &config, + Some("limited-vision"), + None, + 4, + ) + .expect("text-only models receive analyze_image paths instead of a pixel batch"); + } + + fn image_admission_model( + id: &str, + limit: Option, + ) -> crate::service::config::types::AIModelConfig { + use crate::service::config::types::{AIModelConfig, ModelCapability}; + AIModelConfig { + id: id.into(), + name: id.into(), + enabled: true, + capabilities: vec![ + ModelCapability::TextChat, + ModelCapability::ImageUnderstanding, + ], + max_images_per_request: limit, + ..Default::default() + } + } + + #[test] + fn input_image_count_admission_uses_all_automatic_channels_but_only_the_manual_channel() { + use crate::service::config::types::AIConfig; + use openbitfun_config_contracts::channel_routing::ModelChannelRouting; + let mut primary = image_admission_model("primary-vision", Some(5)); + primary.channel_routing = Some(ModelChannelRouting { + fallback_model_ids: vec!["backup-vision".into(), "legacy-vision".into()], + ..Default::default() + }); + let mut config = AIConfig { + models: vec![ + primary, + image_admission_model("backup-vision", Some(3)), + image_admission_model("legacy-vision", None), + ], + ..Default::default() + }; + let error = ConversationCoordinator::validate_input_image_count( + &config, + Some("primary-vision"), + None, + 5, + ) + .expect_err("new user pixels must survive a later fallback"); + assert!(matches!(error, OpenBitFunError::Validation(message) if message.contains("5 > 3"))); + ConversationCoordinator::validate_input_image_count( + &config, + Some("primary-vision"), + None, + 3, + ) + .unwrap(); + + config.models[0] + .channel_routing + .as_mut() + .unwrap() + .manual_channel = Some("primary-vision".into()); + ConversationCoordinator::validate_input_image_count( + &config, + Some("primary-vision"), + None, + 5, + ) + .expect("manual channels do not spill to the smaller backup"); + config.models[1].max_images_per_request = Some(6); + config.models[0] + .channel_routing + .as_mut() + .unwrap() + .manual_channel = Some("backup-vision".into()); + ConversationCoordinator::validate_input_image_count( + &config, + Some("primary-vision"), + None, + 6, + ) + .expect("an unselected primary cannot restrict the manual backup"); + } + + #[test] + fn input_image_count_admission_uses_the_pool_and_member_channel_common_budget() { + use crate::service::config::types::{AIConfig, ModelCapability}; + use openbitfun_config_contracts::channel_routing::ModelChannelRouting; + let mut disabled = image_admission_model("disabled", Some(1)); + disabled.enabled = false; + disabled.channel_routing = Some(ModelChannelRouting { + fallback_model_ids: vec!["unrelated-vision".into()], + ..Default::default() + }); + let mut text_only = image_admission_model("text-only", Some(1)); + text_only.capabilities = vec![ModelCapability::TextChat]; + let mut config = AIConfig { + models: vec![ + image_admission_model("large-vision", Some(5)), + image_admission_model("small-vision", Some(3)), + image_admission_model("legacy-vision", None), + disabled, + text_only, + image_admission_model("unrelated-vision", Some(1)), + image_admission_model("member-backup", Some(2)), + ], + ..Default::default() + }; + config.model_pools.pools.push(serde_json::json!({ + "id": "vision-pool", "name": "Vision pool", "strategy": "balanced", + "member_model_ids": ["large-vision", "small-vision", "legacy-vision", "disabled", "text-only"] + })); + let error = ConversationCoordinator::validate_input_image_count( + &config, + Some("large-vision"), + Some("vision-pool"), + 5, + ) + .expect_err("the first selected member cannot widen the pool budget"); + assert!(matches!(error, OpenBitFunError::Validation(message) if message.contains("5 > 3"))); + ConversationCoordinator::validate_input_image_count( + &config, + Some("large-vision"), + Some("vision-pool"), + 3, + ) + .expect("disabled, unrelated, text-only and unspecified limits do not tighten the pool"); + ConversationCoordinator::validate_input_image_count(&config, None, Some("vision-pool"), 3) + .expect("queue and steering admission need no current pool member"); + + config.models[0].channel_routing = Some(ModelChannelRouting { + fallback_model_ids: vec!["member-backup".into()], + ..Default::default() + }); + let error = ConversationCoordinator::validate_input_image_count( + &config, + Some("small-vision"), + Some("vision-pool"), + 3, + ) + .expect_err("every enabled member's fallback also participates in the common budget"); + assert!(matches!(error, OpenBitFunError::Validation(message) if message.contains("3 > 2"))); + } + #[test] fn interrupted_turn_intent_commit_wins_over_timeout_revocation() { let intents = dashmap::DashMap::new(); diff --git a/src/crates/assembly/core/src/agentic/coordination/scheduler.rs b/src/crates/assembly/core/src/agentic/coordination/scheduler.rs index 081a4ef1a2..85cac4168b 100644 --- a/src/crates/assembly/core/src/agentic/coordination/scheduler.rs +++ b/src/crates/assembly/core/src/agentic/coordination/scheduler.rs @@ -553,16 +553,17 @@ impl DialogScheduler { display_content: Option, mut attachments: Vec, metadata: serde_json::Map, - ) -> Result { + ) -> PortResult { if content.trim().is_empty() && attachments.is_empty() { - return Err("Steering content cannot be empty".to_string()); + return Err(PortError::new( + PortErrorKind::InvalidRequest, + "Steering content cannot be empty", + )); } // Reject a malformed attachment here rather than at the round boundary: // the caller is still holding the user's message and can surface the // failure, whereas the injection consumer would have to drop it. - let mut images = agent_dialog_turn_image_contexts(&attachments) - .map_err(|error| error.to_string())? - .unwrap_or_default(); + let mut images = agent_dialog_turn_image_contexts(&attachments)?.unwrap_or_default(); let _operation_guard = self.lock_session_operation(&session_id).await; let active_turn_id = match self .session_manager @@ -584,7 +585,7 @@ impl DialogScheduler { self.coordinator .prepare_input_images(&session_id, &mut images) .await - .map_err(|error| error.to_string())?; + .map_err(|error| SchedulerSubmitError::Core(error).into_port_error())?; for (attachment, image) in attachments.iter_mut().zip(&images) { if let Some(path) = &image.image_path { attachment @@ -614,14 +615,15 @@ impl DialogScheduler { "Steering rejected: target turn is not running: session_id={}, turn_id={}", session_id, turn_id ); - Err(error) + Err(PortError::new(PortErrorKind::SessionInUse, error)) } DialogSteeringAction::Buffer { mut injection, outcome, } => { self.prepare_goal_steering(&session_id, &turn_id, &mut injection) - .await?; + .await + .map_err(|error| PortError::new(PortErrorKind::SessionInUse, error))?; self.round_injection_buffer.push(&session_id, injection); let DialogSteerOutcome::Buffered { steering_id, .. } = &outcome; info!( @@ -3274,10 +3276,6 @@ impl AgentDialogTurnPort for DialogScheduler { &self, request: AgentDialogSteerRequest, ) -> PortResult { - // An empty-but-attachment-free message and a malformed attachment are - // both bad requests; only a live-turn mismatch means "session in use". - let invalid_request = request.content.trim().is_empty() && request.attachments.is_empty() - || agent_dialog_turn_image_contexts(&request.attachments).is_err(); DialogScheduler::buffer_steering( self, request.session_id, @@ -3288,16 +3286,6 @@ impl AgentDialogTurnPort for DialogScheduler { request.metadata, ) .await - .map_err(|error| { - PortError::new( - if invalid_request { - PortErrorKind::InvalidRequest - } else { - PortErrorKind::SessionInUse - }, - error, - ) - }) } async fn recover_interrupted_turn( @@ -4988,7 +4976,7 @@ mod tests { .await .expect_err("stale processing state must not accept steering"); - assert!(error.contains("no longer running"), "{error}"); + assert!(error.message.contains("no longer running"), "{error}"); assert!(scheduler .round_injection_monitor() .take_pending(session_id, turn_id) @@ -5016,6 +5004,136 @@ mod tests { assert_eq!(error.kind, PortErrorKind::InvalidRequest); } + #[tokio::test] + async fn input_image_count_rejection_preserves_context_and_allows_plain_continuation() { + use crate::service::config::types::ModelCapability; + + for steer in [false, true] { + let (scheduler, sessions, _, root) = test_scheduler(); + let session_id = "limited-image-input"; + let workspace = fixture_workspace_dir(root.path().join("limited-image-input")); + sessions + .create_session_with_id( + Some(session_id.into()), + "Limited image input".into(), + "Standard".into(), + SessionConfig { + workspace_path: Some(workspace.to_string_lossy().into_owned()), + model_id: Some("limited-vision".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + sessions + .update_session_state( + session_id, + SessionState::Processing { + current_turn_id: "active-turn".into(), + phase: ProcessingPhase::Thinking, + }, + ) + .await + .unwrap(); + scheduler + .active_turns + .insert(session_id, desktop_active_turn("active-turn")); + let context_before = + serde_json::to_value(sessions.get_context_messages(session_id).await.unwrap()) + .unwrap(); + let attachments = (0..4) + .map(|index| { + AgentInputAttachment::image_context( + format!("image-{index}"), + Some(format!("/runtime/image-{index}.png")), + None, + "image/png", + None, + ) + }) + .collect::>(); + let config = AIConfig { + models: vec![AIModelConfig { + id: "limited-vision".into(), + name: "Limited vision".into(), + enabled: true, + capabilities: vec![ + ModelCapability::TextChat, + ModelCapability::ImageUnderstanding, + ], + max_images_per_request: Some(3), + ..Default::default() + }], + ..Default::default() + }; + let turn_request = |attachments| AgentDialogTurnRequest { + session_id: session_id.into(), + message: "Continue".into(), + original_message: None, + turn_id: Some("next-turn".into()), + agent_type: "Standard".into(), + workspace_path: None, + workspace_id: None, + remote_connection_id: None, + remote_ssh_host: None, + policy: DialogSubmissionPolicy::for_source(DialogTriggerSource::Cli), + execution: Default::default(), + output_schema: None, + reply_route: None, + prepended_reminders: Vec::new(), + attachments, + metadata: Default::default(), + }; + let error = TEST_MODEL_RESOLUTION_AI_CONFIG + .scope(config, async { + if steer { + scheduler + .steer_dialog_turn(AgentDialogSteerRequest { + session_id: session_id.into(), + turn_id: "active-turn".into(), + content: "Compare these images".into(), + display_content: None, + attachments, + metadata: Default::default(), + }) + .await + .expect_err("oversized steering must stay with its sender") + } else { + scheduler + .submit_dialog_turn(turn_request(attachments)) + .await + .expect_err("oversized input must not enter the queue") + } + }) + .await; + assert_eq!(error.kind, PortErrorKind::InvalidRequest); + assert!(error.message.contains("4 > 3"), "{error}"); + assert_eq!(scheduler.queue_depth(session_id), 0); + assert_eq!( + scheduler.round_injection_buffer.pending_count(session_id), + 0 + ); + assert_eq!( + serde_json::to_value(sessions.get_context_messages(session_id).await.unwrap()) + .unwrap(), + context_before, + "rejecting attachments must leave model history intact" + ); + assert!(sessions + .get_session(session_id) + .unwrap() + .dialog_turn_ids + .is_empty()); + + let outcome = scheduler + .submit_dialog_turn(turn_request(Vec::new())) + .await + .expect("a plain continuation must remain admissible after rejection"); + assert!(matches!(outcome, DialogSubmitOutcome::Queued { .. })); + assert_eq!(scheduler.queue_depth(session_id), 1); + } + } + #[tokio::test] async fn thread_goal_submit_retry_does_not_block_another_sessions_outcome() { let (scheduler, sessions, _, root) = test_scheduler_with_persistence(true); diff --git a/src/crates/assembly/core/src/agentic/execution/AGENTS.md b/src/crates/assembly/core/src/agentic/execution/AGENTS.md index 15d518c054..bfaefb2d38 100644 --- a/src/crates/assembly/core/src/agentic/execution/AGENTS.md +++ b/src/crates/assembly/core/src/agentic/execution/AGENTS.md @@ -296,6 +296,16 @@ and coordinate metadata through the provider wire converters: cargo test -p openbitfun-core --no-default-features --features agent-runtime,git,tools-computer-use --lib computer_use_pixels_and_geometry_reach_real_provider_wire ``` +For configured image limits, legacy tool-image history, plain continuation after +a provider rejection, and user attachment admission: + +```bash +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git --lib -- configured_image_limit input_image_count +``` + +These fixtures use an isolated loopback model and temporary session state. They +do not exercise live providers, SSH, relay, peer, or detached dispatch transports. + ## Background result delivery Portable decisions are owned by `openbitfun-agent-runtime::scheduler::result_delivery`; diff --git a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs index 89082b923f..43aa71bf09 100644 --- a/src/crates/assembly/core/src/agentic/execution/execution_engine.rs +++ b/src/crates/assembly/core/src/agentic/execution/execution_engine.rs @@ -5607,6 +5607,136 @@ mod tests { assert_eq!(image_attachments.unwrap()[0].data_base64, pixels); } + #[tokio::test] + async fn configured_image_limit_recovers_legacy_tool_history_on_plain_continue() { + use crate::agentic::execution::round_executor::tests::{ + retry_test_success, RetryTestServer, + }; + use openbitfun_core_types::errors::{AiProviderError, ErrorCategory}; + + let server = RetryTestServer::new(vec![ + ( + 400, + json!({"error": {"code": "invalid_request_error", "message": "At most 3 images per request"}}).to_string(), + ), + retry_test_success(), + retry_test_success(), + ]); + let image = crate::agentic::image_analysis::attachments::test_image(); + let pixels = image + .data_url + .unwrap() + .split_once(',') + .unwrap() + .1 + .to_owned(); + let mut original = vec![Message::user("Inspect these files".into())]; + for index in 0..4 { + let call = ToolCall { + tool_id: format!("view-{index}"), + tool_name: "view_image".into(), + arguments: json!({"path": format!("image-{index}.png")}), + raw_arguments: None, + is_error: false, + parse_error: None, + recovered_from_truncation: false, + repair_kind: Default::default(), + }; + original.push(Message::assistant_with_tools( + String::new(), + vec![call.clone()], + )); + original.push(Message::tool_result(ToolResult { + tool_id: call.tool_id, + tool_name: "view_image".into(), + effective_tool_name: None, + result: json!({"path": format!("image-{index}.png")}), + result_for_assistant: Some("Image attached for model vision.".into()), + is_error: false, + duration_ms: None, + image_attachments: Some(vec![crate::util::types::ToolImageAttachment { + mime_type: "image/png".into(), + data_base64: pixels.clone(), + }]), + })); + } + // Restore a pre-existing context, including pixels from a failed turn. + let snapshot = serde_json::to_value(&original).unwrap(); + let store = SessionContextStore::new(); + store.add_messages( + "image-limit", + serde_json::from_value(snapshot.clone()).unwrap(), + ); + let uncapped = server.client(); + let mut capped = (*uncapped).clone(); + capped.config.max_images_per_request = Some(3); + + for attempt in 0..3 { + if attempt > 0 { + store.add_message("image-limit", Message::user("Continue".into())); + } + let messages = ExecutionEngine::build_ai_messages_for_send( + &store.get_context_messages("image-limit"), + "openai", + None, + None, + "continued-turn", + true, + &[], + ) + .await + .unwrap(); + let client = if attempt == 0 { + uncapped.as_ref() + } else { + &capped + }; + let result = tokio::time::timeout( + Duration::from_secs(5), + client.send_message_once_with_trace_and_request_context(messages, None, None, None), + ) + .await + .expect("image request must settle"); + if attempt == 0 { + let error = result.expect_err("the uncapped request reproduces provider rejection"); + assert_eq!( + error.downcast_ref::().unwrap().category, + ErrorCategory::InvalidRequest, + ); + } else { + result.expect("plain continuation must fit the configured image limit"); + } + } + + let requests = server.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + for (request, expected) in requests.iter().zip([4, 3, 3]) { + let images = request["messages"] + .as_array() + .unwrap() + .iter() + .filter_map(|message| message["content"].as_array()) + .flatten() + .filter(|part| part["type"] == "image_url") + .count(); + assert_eq!(images, expected, "{request}"); + assert_eq!( + request["messages"] + .as_array() + .unwrap() + .iter() + .filter(|message| message["role"] == "tool") + .count(), + 4 + ); + } + let preserved = store.get_context_messages("image-limit"); + assert_eq!( + serde_json::to_value(&preserved[..original.len()]).unwrap(), + snapshot + ); + } + #[test] fn image_input_requirement_tracks_inherited_steered_and_retained_images() { use super::ImageInputRequirement as Requirement; diff --git a/src/crates/assembly/core/src/agentic/execution/prompt_suggestion.rs b/src/crates/assembly/core/src/agentic/execution/prompt_suggestion.rs index 00048037a7..3cc91f7f14 100644 --- a/src/crates/assembly/core/src/agentic/execution/prompt_suggestion.rs +++ b/src/crates/assembly/core/src/agentic/execution/prompt_suggestion.rs @@ -713,6 +713,7 @@ mod tests { format: "openai".to_string(), context_window: 4096, max_tokens: Some(128), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/agentic/execution/round_executor.rs b/src/crates/assembly/core/src/agentic/execution/round_executor.rs index eef80572e5..7cb233d913 100644 --- a/src/crates/assembly/core/src/agentic/execution/round_executor.rs +++ b/src/crates/assembly/core/src/agentic/execution/round_executor.rs @@ -2947,6 +2947,7 @@ pub(super) mod tests { format: "openai".to_string(), context_window: 4096, max_tokens: Some(128), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/agentic/image_generation/mod.rs b/src/crates/assembly/core/src/agentic/image_generation/mod.rs index a85c2b90e2..027ea4eeae 100644 --- a/src/crates/assembly/core/src/agentic/image_generation/mod.rs +++ b/src/crates/assembly/core/src/agentic/image_generation/mod.rs @@ -119,6 +119,7 @@ fn transient_ai_config(service: &ImageServiceConfig) -> AIConfig { format: String::new(), context_window: 0, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/agentic/memories/service.rs b/src/crates/assembly/core/src/agentic/memories/service.rs index 9d1a3049ba..e575e0eae5 100644 --- a/src/crates/assembly/core/src/agentic/memories/service.rs +++ b/src/crates/assembly/core/src/agentic/memories/service.rs @@ -1000,6 +1000,7 @@ mod tests { format: "openai".to_string(), context_window, max_tokens, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/function_agents/port_adapters.rs b/src/crates/assembly/core/src/function_agents/port_adapters.rs index d5ec2600cd..b36fee4ae4 100644 --- a/src/crates/assembly/core/src/function_agents/port_adapters.rs +++ b/src/crates/assembly/core/src/function_agents/port_adapters.rs @@ -160,6 +160,7 @@ mod tests { format: "openai".to_string(), context_window: 8192, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/infrastructure/ai/client_factory.rs b/src/crates/assembly/core/src/infrastructure/ai/client_factory.rs index 1f220f7715..42ea6d89a0 100644 --- a/src/crates/assembly/core/src/infrastructure/ai/client_factory.rs +++ b/src/crates/assembly/core/src/infrastructure/ai/client_factory.rs @@ -1915,6 +1915,7 @@ mod tests { format: "openai".to_string(), context_window: 4096, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/infrastructure/ai/reasoning_catalog.rs b/src/crates/assembly/core/src/infrastructure/ai/reasoning_catalog.rs index 6140dc2b08..8e6f1b68a8 100644 --- a/src/crates/assembly/core/src/infrastructure/ai/reasoning_catalog.rs +++ b/src/crates/assembly/core/src/infrastructure/ai/reasoning_catalog.rs @@ -595,6 +595,7 @@ mod tests { format: "responses".to_string(), context_window: 128_000, max_tokens: Some(8192), + max_images_per_request: None, temperature: None, top_p: None, inline_think_in_text: false, diff --git a/src/crates/assembly/core/src/service/config/providers.rs b/src/crates/assembly/core/src/service/config/providers.rs index 3b11389461..8f05b31a9d 100644 --- a/src/crates/assembly/core/src/service/config/providers.rs +++ b/src/crates/assembly/core/src/service/config/providers.rs @@ -44,6 +44,8 @@ fn ai_validation_error_location(message: &str) -> (String, String) { Some(("context_window", "MODEL_CONTEXT_WINDOW_INVALID")) } else if message.contains("max_tokens") { Some(("max_tokens", "MODEL_MAX_TOKENS_INVALID")) + } else if message.contains("max_images_per_request") { + Some(("max_images_per_request", "MODEL_MAX_IMAGES_INVALID")) } else if message.contains("temperature") { Some(("temperature", "MODEL_TEMPERATURE_INVALID")) } else if message.contains("top_p") { @@ -1043,6 +1045,25 @@ mod tests { assert_eq!(result.diagnostics[0].path, "ai.models[0].context_window"); } + #[tokio::test] + async fn registry_reports_image_budget_validation_path() { + let mut config = GlobalConfig::default(); + config.ai.models.push(AIModelConfig { + id: "limited".into(), + name: "Limited vision".into(), + provider: "openai".into(), + enabled: true, + max_images_per_request: Some(0), + ..AIModelConfig::default() + }); + let result = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .unwrap(); + assert_eq!(result.errors[0].path, "ai.models[0].max_images_per_request"); + assert_eq!(result.errors[0].code, "MODEL_MAX_IMAGES_INVALID"); + } + #[tokio::test] async fn rejects_default_and_agent_references_to_incapable_models() { let mut config = AIConfig::default(); diff --git a/src/crates/assembly/core/src/service/token_usage/statistics.rs b/src/crates/assembly/core/src/service/token_usage/statistics.rs index 761b26c44f..2711d623f3 100644 --- a/src/crates/assembly/core/src/service/token_usage/statistics.rs +++ b/src/crates/assembly/core/src/service/token_usage/statistics.rs @@ -250,6 +250,7 @@ mod tests { api_key: String::new(), context_window: None, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, enabled: true, diff --git a/src/crates/contracts/config-contracts/src/conversion.rs b/src/crates/contracts/config-contracts/src/conversion.rs index e062b7edcf..41ad3551d2 100644 --- a/src/crates/contracts/config-contracts/src/conversion.rs +++ b/src/crates/contracts/config-contracts/src/conversion.rs @@ -84,6 +84,13 @@ impl TryFrom for AIConfig { )); } + if other.max_images_per_request == Some(0) { + return Err(format!( + "Model '{}' max_images_per_request must be greater than 0", + other.name + )); + } + let custom_request_body = if let Some(body_str) = other .custom_request_body .as_deref() @@ -103,6 +110,12 @@ impl TryFrom for AIConfig { None }; + crate::normalization::validate_image_budget_request_body( + other.max_images_per_request, + custom_request_body.as_ref(), + ) + .map_err(|error| format!("Model '{}' {error}", other.name))?; + let request_url = other .request_url .clone() @@ -155,6 +168,7 @@ impl TryFrom for AIConfig { format: other.provider.clone(), context_window, max_tokens: Some(max_tokens), + max_images_per_request: other.max_images_per_request, temperature: other.temperature, top_p: other.top_p, inline_think_in_text: other.inline_think_in_text, @@ -251,6 +265,7 @@ mod tests { api_key: "key".to_string(), context_window: Some(128000), max_tokens: Some(4096), + max_images_per_request: None, temperature: None, top_p: None, enabled: true, @@ -276,6 +291,90 @@ mod tests { assert_eq!(config.model, "test-model"); } + #[test] + fn image_budget_preserves_legacy_payloads_and_runtime_round_trips() { + let legacy_payload = serde_json::to_value(base_model_config()).unwrap(); + assert!(legacy_payload.get("max_images_per_request").is_none()); + let legacy: AIModelConfig = serde_json::from_value(legacy_payload.clone()).unwrap(); + assert_eq!(legacy.max_images_per_request, None); + assert_eq!(serde_json::to_value(&legacy).unwrap(), legacy_payload); + let fingerprint = crate::types::model_runtime_binding_fingerprint(&legacy); + let mut limited = legacy.clone(); + limited.max_images_per_request = Some(3); + let saved = serde_json::to_value(&limited).unwrap(); + assert_eq!(saved["max_images_per_request"], 3); + let loaded: AIModelConfig = serde_json::from_value(saved).unwrap(); + let runtime = AIConfig::try_from(loaded).unwrap(); + assert_eq!(runtime.max_images_per_request, Some(3)); + let runtime_payload = serde_json::to_value(runtime).unwrap(); + let restored: AIConfig = serde_json::from_value(runtime_payload).unwrap(); + assert_eq!(restored.max_images_per_request, Some(3)); + assert_ne!( + crate::types::model_runtime_binding_fingerprint(&limited), + fingerprint + ); + limited.max_images_per_request = None; + assert_eq!( + crate::types::model_runtime_binding_fingerprint(&limited), + fingerprint + ); + + let runtime = AIConfig::try_from(legacy).unwrap(); + let payload = serde_json::to_value(runtime).unwrap(); + assert!(payload.get("max_images_per_request").is_none()); + let restored: AIConfig = serde_json::from_value(payload).unwrap(); + assert_eq!(restored.max_images_per_request, None); + } + + #[test] + fn image_budget_rejects_zero_without_discarding_model_configuration() { + let mut model = base_model_config(); + model.max_images_per_request = Some(0); + assert!(crate::normalization::validate_model_config(&model, 0) + .unwrap_err() + .contains("max_images_per_request")); + assert!(AIConfig::try_from(model.clone()) + .unwrap_err() + .contains("max_images_per_request")); + let mut config = crate::GlobalConfig::default(); + config.ai.models.push(model.clone()); + crate::normalization::recover_persisted_config(&mut config); + assert_eq!(config.ai.models.len(), 1); + assert!(!config.ai.models[0].enabled); + assert_eq!(config.ai.models[0].api_key, model.api_key); + assert_eq!(config.ai.models[0].max_images_per_request, Some(0)); + } + + #[test] + fn image_budget_rejects_wire_message_overrides_but_keeps_other_custom_options() { + for key in ["messages", "input", "contents"] { + let mut model = base_model_config(); + model.custom_request_body = Some(serde_json::json!({key: []}).to_string()); + assert!( + AIConfig::try_from(model.clone()).is_ok(), + "legacy override: {key}" + ); + model.max_images_per_request = Some(3); + assert!(crate::normalization::validate_model_config(&model, 0) + .unwrap_err() + .contains("custom_request_body")); + assert!(AIConfig::try_from(model) + .unwrap_err() + .contains("custom_request_body")); + } + let mut model = base_model_config(); + model.max_images_per_request = Some(3); + model.custom_request_body = Some(r#"{"temperature":0.2}"#.into()); + assert!(crate::normalization::validate_model_config(&model, 0).is_ok()); + assert_eq!( + AIConfig::try_from(model) + .unwrap() + .custom_request_body + .unwrap()["temperature"], + 0.2 + ); + } + #[test] fn accepts_legacy_missing_null_and_blank_custom_request_bodies() { for body in [ diff --git a/src/crates/contracts/config-contracts/src/normalization.rs b/src/crates/contracts/config-contracts/src/normalization.rs index 9b3cb6715f..494aa0806a 100644 --- a/src/crates/contracts/config-contracts/src/normalization.rs +++ b/src/crates/contracts/config-contracts/src/normalization.rs @@ -4,6 +4,26 @@ use crate::types::{ }; use std::collections::HashSet; +/// Custom wire messages bypass the host's image selection and cannot be mixed +/// with a host-enforced image budget. Other request options remain editable. +pub(crate) fn validate_image_budget_request_body( + max_images_per_request: Option, + body: Option<&serde_json::Value>, +) -> Result<(), &'static str> { + if max_images_per_request.is_some() + && body + .and_then(serde_json::Value::as_object) + .is_some_and(|body| { + ["messages", "input", "contents"] + .iter() + .any(|key| body.contains_key(*key)) + }) + { + return Err("custom_request_body cannot override messages, input, or contents when max_images_per_request is configured"); + } + Ok(()) +} + /// Provider-neutral model validation, shared by startup and offline import. pub fn validate_model_config(model: &crate::AIModelConfig, index: usize) -> Result<(), String> { if !model.supports_text_generation() { @@ -24,6 +44,20 @@ pub fn validate_model_config(model: &crate::AIModelConfig, index: usize) -> Resu if !model.enabled { return Ok(()); } + if model.max_images_per_request == Some(0) { + return Err(format!( + "Model '{}' max_images_per_request must be greater than 0 at index {index}", + model.name + )); + } + if model.max_images_per_request.is_some() { + let body = model + .custom_request_body + .as_deref() + .and_then(|body| serde_json::from_str::(body).ok()); + validate_image_budget_request_body(model.max_images_per_request, body.as_ref()) + .map_err(|error| format!("Model '{}' {error} at index {index}", model.name))?; + } if model.name.trim().is_empty() { return Err(format!("Model name is required at index {index}")); } diff --git a/src/crates/contracts/config-contracts/src/types.rs b/src/crates/contracts/config-contracts/src/types.rs index 33b2e5e06b..c6b3da6038 100644 --- a/src/crates/contracts/config-contracts/src/types.rs +++ b/src/crates/contracts/config-contracts/src/types.rs @@ -1821,6 +1821,10 @@ pub struct AIModelConfig { /// Optional advanced override for the request output limit. When absent, /// OpenBitFun derives a tiered limit from the context window at runtime. pub max_tokens: Option, + /// Optional image input ceiling across the complete request, including history + /// and tool results. Missing preserves the existing provider behavior. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_images_per_request: Option, pub temperature: Option, pub top_p: Option, pub enabled: bool, @@ -2310,6 +2314,7 @@ impl Default for AIModelConfig { api_key: String::new(), context_window: None, max_tokens: None, + max_images_per_request: None, temperature: None, top_p: None, enabled: false, diff --git a/src/crates/contracts/core-types/src/ai.rs b/src/crates/contracts/core-types/src/ai.rs index 26494362e4..3721fbb607 100644 --- a/src/crates/contracts/core-types/src/ai.rs +++ b/src/crates/contracts/core-types/src/ai.rs @@ -637,6 +637,9 @@ pub struct AIConfig { pub format: String, pub context_window: u32, pub max_tokens: Option, + /// Optional host-side image input ceiling across the complete request. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_images_per_request: Option, pub temperature: Option, pub top_p: Option, pub inline_think_in_text: bool, diff --git a/src/crates/interfaces/app-server/src/management/owner.rs b/src/crates/interfaces/app-server/src/management/owner.rs index 0c675ce96f..ffc38f7308 100644 --- a/src/crates/interfaces/app-server/src/management/owner.rs +++ b/src/crates/interfaces/app-server/src/management/owner.rs @@ -441,6 +441,7 @@ fn model_from_mutation( api_key, context_window: mutation.context_window, max_tokens: mutation.max_tokens, + max_images_per_request: current.max_images_per_request, temperature: current.temperature, top_p: current.top_p, enabled: mutation.enabled, @@ -1891,6 +1892,7 @@ mod tests { fn model_mutation_preserves_and_replaces_write_only_values() { let existing = openbitfun_core::service::config::AIModelConfig { api_key: "existing-key".to_string(), + max_images_per_request: Some(3), custom_headers: Some(HashMap::from([( "Authorization".to_string(), "existing-header".to_string(), @@ -1908,6 +1910,7 @@ mod tests { let preserved = model_from_mutation(mutation(), Some(existing.clone())) .expect("preserve existing model secrets"); assert_eq!(preserved.api_key, "existing-key"); + assert_eq!(preserved.max_images_per_request, Some(3)); assert_eq!(preserved.custom_headers, existing.custom_headers); assert_eq!( preserved.channel_routing, existing.channel_routing, @@ -1926,6 +1929,7 @@ mod tests { let replaced = model_from_mutation(replacement, Some(existing)).expect("replace model secrets"); assert_eq!(replaced.api_key, "new-key"); + assert_eq!(replaced.max_images_per_request, Some(3)); assert!(replaced.custom_headers.is_none()); assert_eq!(replaced.custom_request_body.as_deref(), Some("new-body")); assert_eq!(replaced.channel_routing, original_route); diff --git a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx index af016628ec..5cd3f24515 100644 --- a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx +++ b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx @@ -173,6 +173,42 @@ describe('ModelSettingsPage loading and provider actions', () => { expect(configManager.getConfig).not.toHaveBeenCalled(); }); + it.each([undefined, 3])('saves and clears the per-model image ceiling (previous: %s)', async previous => { + const configured: AIModelConfig = { + ...model, api_key: 'fixture-key', context_window: 128_000, + max_images_per_request: previous, + capabilities: ['text_chat', 'image_understanding'], + }; + const models = [configured]; + const values: Record = { + 'ai.models': models, 'ai.default_models': { primary: configured.id }, + 'ai.proxy': { enabled: false, url: '', username: '', password: '' }, + 'ai.stream_idle_timeout_secs': 120, 'ai.stream_ttft_timeout_secs': 60, + }; + vi.spyOn(configManager, 'hasCachedConfig').mockImplementation(path => path in values); + vi.spyOn(configManager, 'getCachedConfig').mockImplementation((path: string) => values[path] as T); + const update = vi.spyOn(configManager, 'updateConfig').mockImplementation( + async (_path: string, transform: (current: T) => T) => transform(models as T), + ); + vi.spyOn(aiApi, 'testAIConfigConnection').mockImplementation(() => new Promise(() => {})); + await act(async () => root.render()); + await act(async () => container.querySelector( + `[data-config-id="${configured.id}"] button[aria-label="${copy('actions.edit')}"]`, + )!.click()); + const editor = document.querySelector('.openbitfun-model-settings__editor-dialog')!; + const input = editor.querySelector('[data-testid="settings-model-image-limit"]')!; + expect(input.value).toBe(previous === undefined ? '' : '3'); + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call(input, previous === undefined ? '3' : ''); + input.dispatchEvent(new Event('input', { bubbles: true })); + }); + await act(async () => editor.querySelector('[data-testid="settings-model-save-btn"]')!.click()); + expect(update).toHaveBeenCalledOnce(); + const saved = await update.mock.results[0].value as AIModelConfig[]; + expect(saved[0].max_images_per_request).toBe(previous === undefined ? 3 : undefined); + expect(saved[0].api_key).toBe(configured.api_key); + }); + it.each(['add', 'remove', 'reorder', 'manual'] as const)('protects a channel-only %s edit and saves its original model snapshot', async operation => { const primary: AIModelConfig = { ...model, api_key: 'fixture-key', context_window: 128_000, max_tokens: 8_000, diff --git a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx index 91291e51c5..1a966f9b25 100644 --- a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx +++ b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx @@ -3034,6 +3034,30 @@ const ModelSettingsPage: React.FC = () => { {modelRoute?.request_url || resolveRequestUrl(editingConfig.base_url || '', manualFormat, draft.modelName)} , )} + {renderModelField( + t('form.maxImagesPerRequest'), + { + const value = event.target.value; + const count = Number(value); + if (value === '' || (Number.isSafeInteger(count) && count > 0)) { + setSelectedModelDrafts(drafts => updateModelEditorRequestSettings( + drafts, + { max_images_per_request: value === '' ? undefined : count }, + draft.key, + )); + } + }} + />, + { hint: t('form.maxImagesPerRequestHint') }, + )} = ({ status: 'loading', canSend: false, }); + const maxImageCount = composerImageLimit(modelAvailability.maxImagesPerRequest); + const admitComposerImages = useCallback((submittedContexts: readonly ContextItem[]) => { + if (submittedContexts.filter(context => context.type === 'image').length <= maxImageCount) return true; + notificationService.warning(t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }); + return false; + }, [maxImageCount, t]); const isAssistantWorkspace = useMemo( () => resolveSessionAssistantWorkspace({ currentWorkspace: workspace, @@ -3103,7 +3109,10 @@ export const ChatInput: React.FC = ({ let imgCount = currentImageCount; for (const block of params.content) { if (block.type === 'image') { - if (imgCount >= CHAT_INPUT_CONFIG.image.maxCount) break; + if (imgCount >= maxImageCount) { + notificationService.warning(t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }); + break; + } try { const mimeType = block.mimeType || 'image/png'; const binaryString = atob(block.data); @@ -3147,7 +3156,7 @@ export const ChatInput: React.FC = ({ return () => { globalEventBus.off('mcp-app:message', handleMcpAppMessage); }; - }, [addContext, clearPendingLargePastes, conversationScope, currentImageCount, dispatchInput]); + }, [addContext, clearPendingLargePastes, conversationScope, currentImageCount, dispatchInput, maxImageCount]); React.useEffect(() => { const handleInsertContextTag = (event: Event) => { @@ -3647,6 +3656,7 @@ export const ChatInput: React.FC = ({ const messageCharCount = getCharacterCount(message); const question = stripSlashCommand(message, '/btw').trim(); const imagesForBtw = [...imageContexts]; + if (!admitComposerImages(imagesForBtw)) return; // Clear input without adding to main history. dispatchInput({ type: 'CLEAR_VALUE' }); @@ -3704,7 +3714,7 @@ export const ChatInput: React.FC = ({ replacePendingLargePastes(originalPendingLargePastes); dispatchInput({ type: 'SET_VALUE', payload: originalMessage }); } - }, [clearPendingLargePastes, currentSessionId, derivedState, dispatchInput, expandComposerSpecialTokens, imageContexts, inputState.value, isBtwSession, removeContext, replacePendingLargePastes, sessionBoundWorkspacePath, selectTargetSession, setQueuedInput, t]); + }, [admitComposerImages, clearPendingLargePastes, currentSessionId, derivedState, dispatchInput, expandComposerSpecialTokens, imageContexts, inputState.value, isBtwSession, removeContext, replacePendingLargePastes, sessionBoundWorkspacePath, selectTargetSession, setQueuedInput, t]); const submitCompactFromInput = useCallback(async () => { if (!effectiveTargetSessionId || !effectiveTargetSession) { @@ -4164,6 +4174,7 @@ export const ChatInput: React.FC = ({ const originalPendingLargePastes = { ...pendingLargePastesRef.current }; const submittedContexts = [...contexts]; + if (!admitComposerImages(submittedContexts)) return; let clearedRevision: number | undefined; let clearedStoredDraft: ReturnType['drafts'][string] | undefined; @@ -4239,6 +4250,7 @@ export const ChatInput: React.FC = ({ ); } }, [ + admitComposerImages, clearPendingLargePastes, clearContexts, addToHistory, @@ -4492,6 +4504,7 @@ export const ChatInput: React.FC = ({ ); return true; } + if (!admitComposerImages(contexts)) return true; if (!(await confirmPromptCacheGuardIfNeeded())) { return true; } @@ -4564,6 +4577,7 @@ export const ChatInput: React.FC = ({ } return true; }, [ + admitComposerImages, addToHistory, clearPendingLargePastes, isTargetCurrent, @@ -4889,7 +4903,7 @@ export const ChatInput: React.FC = ({ const imageCount = contextStore.getState().contexts .filter(context => context.type === 'image') .length; - if (imageCount >= CHAT_INPUT_CONFIG.image.maxCount) { + if (imageCount >= maxImageCount) { limitReached = true; continue; } @@ -4900,7 +4914,7 @@ export const ChatInput: React.FC = ({ const latestImageCount = contextStore.getState().contexts .filter(context => context.type === 'image') .length; - if (latestImageCount >= CHAT_INPUT_CONFIG.image.maxCount) { + if (latestImageCount >= maxImageCount) { limitReached = true; continue; } @@ -4917,11 +4931,11 @@ export const ChatInput: React.FC = ({ if (limitReached && isExternalFileIntakeRequestCurrent(request)) { notificationService.warning( - t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }), + t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }, ); } - }, [addContext, contextStore, isExternalFileIntakeRequestCurrent, t]); + }, [addContext, contextStore, isExternalFileIntakeRequestCurrent, maxImageCount, t]); /** * Host-side clipboard image read for engines that deliver paste events with @@ -4977,7 +4991,7 @@ export const ChatInput: React.FC = ({ paths, existingContexts: contextStore.getState().contexts, workspacePath: sessionBoundWorkspacePath || undefined, - maxImageCount: CHAT_INPUT_CONFIG.image.maxCount, + maxImageCount, loadMetadata: pathToInspect => workspaceAPI.getFileMetadata(pathToInspect), }); if (!isExternalFileIntakeRequestCurrent(request)) return; @@ -4993,7 +5007,7 @@ export const ChatInput: React.FC = ({ const imageLimitCount = result.failures.filter(failure => failure.reason === 'image-limit').length; notificationService.warning( imageLimitCount === result.failures.length - ? t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }) + ? t('input.maxImagesWarning', { count: maxImageCount }) : t('input.externalFiles.partialFailure', { failed: result.failures.length, added: result.contexts.length, @@ -5006,6 +5020,7 @@ export const ChatInput: React.FC = ({ contextStore, isExternalFileIntakeRequestCurrent, sessionBoundWorkspacePath, + maxImageCount, t, ]); @@ -5408,6 +5423,10 @@ export const ChatInput: React.FC = ({ // cannot block cancellation or other controls that do not start a turn. if (!modelAvailability.canSend) return; + // A model switch can lower the limit after images have entered the draft. + // Preserve that draft so the user can choose which attachments to remove. + if (!admitComposerImages(submittedContexts)) return; + if (draftWorkspaceId && !draftTargetWorkspace) { notificationService.error(t('workspaceStrip.unavailable')); return; @@ -5502,6 +5521,7 @@ export const ChatInput: React.FC = ({ }, [ draftWorkspaceId, draftTargetWorkspace, + admitComposerImages, isModelSwitching, modelAvailability.canSend, targetCanSubmit, @@ -6008,9 +6028,9 @@ export const ChatInput: React.FC = ({ }, []); const handleImageInput = useCallback(() => { - const remaining = CHAT_INPUT_CONFIG.image.maxCount - currentImageCount; + const remaining = maxImageCount - currentImageCount; if (remaining <= 0) { - notificationService.warning(t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }), { duration: 3000 }); + notificationService.warning(t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }); return; } @@ -6047,7 +6067,7 @@ export const ChatInput: React.FC = ({ const fileArray = Array.from(files).slice(0, remaining); if (files.length > remaining) { - notificationService.warning(t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }), { duration: 3000 }); + notificationService.warning(t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }); } for (const file of fileArray) { @@ -6066,7 +6086,7 @@ export const ChatInput: React.FC = ({ document.body.appendChild(input); input.click(); - }, [addContext, currentImageCount, t]); + }, [addContext, currentImageCount, maxImageCount, t]); const focusRichTextInputSoon = useCallback(() => { @@ -6423,8 +6443,8 @@ export const ChatInput: React.FC = ({ : undefined } onContextAdded={(context) => { - if (context.type === 'image' && currentImageCount >= CHAT_INPUT_CONFIG.image.maxCount) { - notificationService.warning(t('input.maxImagesWarning', { count: CHAT_INPUT_CONFIG.image.maxCount }), { duration: 3000 }); + if (context.type === 'image' && currentImageCount >= maxImageCount) { + notificationService.warning(t('input.maxImagesWarning', { count: maxImageCount }), { duration: 3000 }); return; } // Images are shown as separate thumbnails outside the editor; they diff --git a/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts index e5a89840f2..8c8382b61f 100644 --- a/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts +++ b/src/web-ui/src/flow_chat/components/ChatInputImageIntake.test.ts @@ -10,6 +10,20 @@ function readChatInputComponent(): string { describe('composer image file intake', () => { const component = readChatInputComponent(); + it.each([ + ['handleSendOrCancel', 'submittedContexts', 'clearComposerForSubmission'], + ['submitBtwFromInput', 'imagesForBtw', "dispatchInput({ type: 'CLEAR_VALUE'"], + ['submitMcpPromptFromInput', 'submittedContexts', 'clearComposerForSubmission'], + ['submitExternalPromptCommandFromInput', 'contexts', "dispatchInput({ type: 'CLEAR_VALUE'"], + ])('keeps an over-limit draft intact in %s after switching models', (entry, snapshot, clear) => { + const start = component.indexOf(`const ${entry} = useCallback(`); + expect(start).toBeGreaterThan(-1); + const body = component.slice(start, component.indexOf(' }, [', start)); + const admission = body.indexOf(`if (!admitComposerImages(${snapshot})) return`); + expect(admission).toBeGreaterThan(-1); + expect(body.indexOf(clear)).toBeGreaterThan(admission); + }); + it('mounts the picker before clicking it so WebKitGTK delivers change', () => { // WebKitGTK (the Linux webview) never fires `change` on a detached file // input after the native chooser closes, which silently dropped every diff --git a/src/web-ui/src/flow_chat/components/ModelSelector.tsx b/src/web-ui/src/flow_chat/components/ModelSelector.tsx index 332c663f31..d95e994111 100644 --- a/src/web-ui/src/flow_chat/components/ModelSelector.tsx +++ b/src/web-ui/src/flow_chat/components/ModelSelector.tsx @@ -19,6 +19,7 @@ import { modelSelectionsConfigPath, effectiveModelSelections, rememberModeSelect import { getRecentModelPool, setRecentModelPool, isSelectablePool, poolForModeSwitch } from '../utils/recentModelPool'; import { modelPoolAPI, readableModelPools, supportsModelPools, supportsBtwModelPools, type ModelPoolDefinition, type ModelPoolMemberStatus } from '@/infrastructure/api/service-api/ModelPoolAPI'; import { getModelPoolContextBudget } from '../utils/modelPoolContextBudget'; +import { modelSelectionImageLimit } from '../utils/modelImageBudget'; import { agentAPI } from '@/infrastructure/api/service-api/AgentAPI'; import { type AIModelCatalog, @@ -107,6 +108,7 @@ export interface ExternalModelSelection { export interface ModelSelectorAvailability { status: ModelAvailabilityStatus; canSend: boolean; + maxImagesPerRequest?: number; } interface ModelSelectorProps { @@ -999,9 +1001,11 @@ export const ModelSelector: React.FC = ({ const availablePoolModelIds = useMemo(() => new Set(availableModels.map(model => model.id)), [availableModels]); const canSelectPool = (pool: ModelPoolDefinition) => isSelectablePool(pool, availablePoolModelIds); const nativeAvailability = useMemo((): ModelSelectorAvailability => { + const imageLimit = modelSelectionImageLimit(allModels, nativeModelResolution.model, poolId ? selectedPool?.member_model_ids ?? [] : undefined); + const imageBudget = imageLimit === undefined ? {} : { maxImagesPerRequest: imageLimit }; if (poolId) { const ready = poolSupported && !poolError && Boolean(selectedPool && isSelectablePool(selectedPool, availablePoolModelIds)); - return { status: poolsLoading ? 'loading' : ready ? 'ready' : 'target-model-unavailable', canSend: ready }; + return { status: poolsLoading ? 'loading' : ready ? 'ready' : 'target-model-unavailable', canSend: ready, ...imageBudget }; } let status: ModelAvailabilityStatus; if (configLoadState === 'loading') { @@ -1025,9 +1029,10 @@ export const ModelSelector: React.FC = ({ return { status, canSend: configLoadState === 'ready' && nativeModelResolution.model !== null, + ...imageBudget, }; }, [ - allModels.length, + allModels, availableModels.length, catalogLoadState, configLoadState, diff --git a/src/web-ui/src/flow_chat/components/ModelSelectorProviderLevels.test.tsx b/src/web-ui/src/flow_chat/components/ModelSelectorProviderLevels.test.tsx index aa30c56c85..5ab258e4df 100644 --- a/src/web-ui/src/flow_chat/components/ModelSelectorProviderLevels.test.tsx +++ b/src/web-ui/src/flow_chat/components/ModelSelectorProviderLevels.test.tsx @@ -1515,6 +1515,33 @@ describe('ModelSelector provider levels', () => { }); }); + it('reports the narrowest image ceiling across a primary selection and its fallback', async () => { + const onAvailabilityChange = vi.fn(); + vi.mocked(configManager.getConfigs).mockResolvedValue({ + 'ai.models': [{ + ...model('limited-vision', 'Limited', 'provider-local', 'https://local.test/v1'), + capabilities: ['text_chat', 'image_understanding'], + max_images_per_request: 5, + channel_routing: { version: 1, fallback_model_ids: ['backup-vision'] }, + }, { + ...model('backup-vision', 'Backup', 'provider-backup', 'https://backup.test/v1'), + capabilities: ['text_chat', 'image_understanding'], + max_images_per_request: 3, + }], + 'ai.default_models': { primary: 'limited-vision' }, + 'ai.agent_model_selections': { mode: { kind: 'model', model_ref: 'primary' }, subagents: { default: { kind: 'model', model_ref: 'fast' }, builtin: {}, fork: { kind: 'inherit' } } }, + }); + await act(async () => { + root.render(); + await Promise.resolve(); + }); + expect(onAvailabilityChange).toHaveBeenLastCalledWith({ + status: 'ready', + canSend: true, + maxImagesPerRequest: 3, + }); + }); + it('treats empty capabilities as the category default for legacy chat models', async () => { const onAvailabilityChange = vi.fn(); vi.mocked(configManager.getConfigs).mockResolvedValue({ diff --git a/src/web-ui/src/flow_chat/constants/chatInputConfig.test.ts b/src/web-ui/src/flow_chat/constants/chatInputConfig.test.ts new file mode 100644 index 0000000000..3164792f36 --- /dev/null +++ b/src/web-ui/src/flow_chat/constants/chatInputConfig.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest'; +import { composerImageLimit } from './chatInputConfig'; + +describe('composer image admission', () => { + it('uses the resolved model ceiling within the existing attachment budget', () => { + expect(composerImageLimit(3)).toBe(3); + expect(composerImageLimit(1)).toBe(1); + expect(composerImageLimit(20)).toBe(5); + }); + + it('keeps the existing attachment budget for older or unusable model facts', () => { + for (const limit of [undefined, 0, -1, 1.5, NaN, Infinity]) { + expect(composerImageLimit(limit)).toBe(5); + } + }); +}); diff --git a/src/web-ui/src/flow_chat/constants/chatInputConfig.ts b/src/web-ui/src/flow_chat/constants/chatInputConfig.ts index 02df943e8e..e48b9ef37b 100644 --- a/src/web-ui/src/flow_chat/constants/chatInputConfig.ts +++ b/src/web-ui/src/flow_chat/constants/chatInputConfig.ts @@ -19,3 +19,10 @@ export const CHAT_INPUT_CONFIG = { syncDelay: 200, }, } as const; + +/** Composer admission only; the executing host enforces the complete request budget. */ +export function composerImageLimit(maxImagesPerRequest?: number): number { + return typeof maxImagesPerRequest === 'number' && Number.isSafeInteger(maxImagesPerRequest) && maxImagesPerRequest > 0 + ? Math.min(CHAT_INPUT_CONFIG.image.maxCount, maxImagesPerRequest) + : CHAT_INPUT_CONFIG.image.maxCount; +} diff --git a/src/web-ui/src/flow_chat/utils/modelImageBudget.test.ts b/src/web-ui/src/flow_chat/utils/modelImageBudget.test.ts new file mode 100644 index 0000000000..2ec34de656 --- /dev/null +++ b/src/web-ui/src/flow_chat/utils/modelImageBudget.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import type { AIModelConfig } from '@/infrastructure/config/types'; +import { modelSelectionImageLimit } from './modelImageBudget'; + +function model(id: string, max_images_per_request?: number, overrides: Partial = {}): AIModelConfig { + return { + id, name: id, model_name: id, provider: 'openai', base_url: 'https://example.test', + enabled: true, category: 'multimodal', capabilities: [], max_images_per_request, ...overrides, + }; +} + +describe('image admission across model selection candidates', () => { + it('uses the narrowest explicit native-image ceiling across automatic fallback routes', () => { + const primary = model('primary', 5, { channel_routing: { version: 1, fallback_model_ids: ['backup', 'legacy'] } }); + expect(modelSelectionImageLimit([primary, model('backup', 3), model('legacy')], primary)).toBe(3); + }); + + it('keeps a manually fixed channel independent of the other route ceilings', () => { + const primary = model('primary', 5, { channel_routing: { + version: 1, fallback_model_ids: ['backup'], manual_channel: 'primary', + } }); + const models = [primary, model('backup', 3)]; + expect(modelSelectionImageLimit(models, primary)).toBe(5); + primary.channel_routing!.manual_channel = 'backup'; + expect(modelSelectionImageLimit(models, primary)).toBe(3); + }); + + it('includes all enabled native pool candidates and their automatic fallback routes', () => { + const routed = model('routed', 5, { channel_routing: { version: 1, fallback_model_ids: ['backup'] } }); + const models = [routed, model('backup', 3), model('legacy'), model('disabled', 1, { + enabled: false, channel_routing: { version: 1, fallback_model_ids: ['unrelated'] }, + }), + model('text', 1, { category: 'general_chat', capabilities: ['text_chat'] }), model('unrelated', 1)]; + expect(modelSelectionImageLimit(models, undefined, ['routed', 'legacy', 'disabled', 'text'])).toBe(3); + expect(modelSelectionImageLimit(models, undefined, ['legacy', 'disabled', 'text'])).toBeUndefined(); + }); + + it('preserves legacy selections and does not use another configured model as fallback', () => { + const primary = model('primary'); + expect(modelSelectionImageLimit([primary, model('unrelated', 1)], primary)).toBeUndefined(); + expect(modelSelectionImageLimit([model('unrelated', 1)])).toBeUndefined(); + }); +}); diff --git a/src/web-ui/src/flow_chat/utils/modelImageBudget.ts b/src/web-ui/src/flow_chat/utils/modelImageBudget.ts new file mode 100644 index 0000000000..e728989401 --- /dev/null +++ b/src/web-ui/src/flow_chat/utils/modelImageBudget.ts @@ -0,0 +1,19 @@ +import type { AIModelConfig } from '@/infrastructure/config/types'; +import { getEffectiveModelCapabilities, isSelectableTextChatModel } from '@/infrastructure/config/services/modelCategory'; + +/** Every native image route must accept the draft before automatic selection. */ +export function modelSelectionImageLimit( + models: readonly AIModelConfig[], + primary?: AIModelConfig | null, + poolMemberIds?: readonly string[], +): number | undefined { + const roots = poolMemberIds ? models.filter(model => model.id && poolMemberIds.includes(model.id)) : primary ? [primary] : []; + const candidateIds = roots.filter(isSelectableTextChatModel).flatMap(model => model.channel_routing?.manual_channel + ? [model.channel_routing.manual_channel] + : [model.id, ...(model.channel_routing?.fallback_model_ids ?? [])]); + const limits = models.filter(model => candidateIds.includes(model.id) + && isSelectableTextChatModel(model) && getEffectiveModelCapabilities(model).includes('image_understanding')) + .map(model => model.max_images_per_request) + .filter((limit): limit is number => typeof limit === 'number' && Number.isSafeInteger(limit) && limit > 0); + return limits.length > 0 ? Math.min(...limits) : undefined; +} diff --git a/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.test.ts b/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.test.ts index a47f759f1d..887a43f26c 100644 --- a/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.test.ts +++ b/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.test.ts @@ -2,6 +2,20 @@ import { describe, expect, it } from 'vitest'; import { getModelEditorRequestSettings, prepareModelEditorRequestSettingsForSave, updateModelEditorRequestSettings } from './modelEditorRequestSettings'; describe('model editor request drafts', () => { + it('keeps image limits per model and clears the optional value without changing legacy defaults', () => { + const drafts = [ + { key: 'limited', requestSettings: getModelEditorRequestSettings({ max_images_per_request: 3 }) }, + { key: 'legacy', requestSettings: getModelEditorRequestSettings() }, + ]; + expect(prepareModelEditorRequestSettingsForSave(drafts[0].requestSettings).max_images_per_request).toBe(3); + const edited = updateModelEditorRequestSettings(drafts, { max_images_per_request: 2 }, 'legacy'); + expect(edited[0].requestSettings.max_images_per_request).toBe(3); + expect(edited[1].requestSettings.max_images_per_request).toBe(2); + const cleared = updateModelEditorRequestSettings(edited, { max_images_per_request: undefined }, 'limited'); + expect(JSON.parse(JSON.stringify(prepareModelEditorRequestSettingsForSave(cleared[0].requestSettings)))) + .not.toHaveProperty('max_images_per_request'); + expect(drafts[1].requestSettings.max_images_per_request).toBeUndefined(); + }); it.each(['', ' \n\t '])('clears a previously saved custom body when the editor contains %j', body => { const persisted = { custom_request_body: '{"temperature":0.2}', custom_request_body_mode: 'trim' as const }; const drafts = [{ key: 'model', requestSettings: getModelEditorRequestSettings(persisted) }]; diff --git a/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.ts b/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.ts index 004ca9b894..cf81fd3696 100644 --- a/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.ts +++ b/src/web-ui/src/infrastructure/config/components/modelEditorRequestSettings.ts @@ -2,6 +2,7 @@ import type { AIModelConfig } from '../types'; export type ModelEditorRequestSettings = Pick = {}): ModelEditorRequestSettings { return { inline_think_in_text: config.inline_think_in_text ?? true, + max_images_per_request: config.max_images_per_request, custom_headers: config.custom_headers ? { ...config.custom_headers } : undefined, custom_headers_mode: config.custom_headers_mode, skip_ssl_verify: config.skip_ssl_verify ?? false, diff --git a/src/web-ui/src/infrastructure/config/types/index.ts b/src/web-ui/src/infrastructure/config/types/index.ts index 44bdd0d343..3ec3d14a0d 100644 --- a/src/web-ui/src/infrastructure/config/types/index.ts +++ b/src/web-ui/src/infrastructure/config/types/index.ts @@ -287,6 +287,8 @@ export interface AIModelConfig { model_name: string; context_window?: number; max_tokens?: number; + /** Optional image input ceiling across history, attachments, and tool results. */ + max_images_per_request?: number; temperature?: number; top_p?: number; enabled: boolean; diff --git a/src/web-ui/src/locales/en-US/settings/models.json b/src/web-ui/src/locales/en-US/settings/models.json index 8d85ea012e..6e95e02121 100644 --- a/src/web-ui/src/locales/en-US/settings/models.json +++ b/src/web-ui/src/locales/en-US/settings/models.json @@ -393,6 +393,8 @@ "provider": "Request Format", "providerPlaceholder": "Select request format", "contextWindow": "Context Window Size", + "maxImagesPerRequest": "Images per request", + "maxImagesPerRequestHint": "Optional maximum image count, including conversation history and tool results. Leave blank to keep the existing behavior.", "contextWindowHint": "Determines when context is compacted; may be lower than the model's supported limit", "contextWindowLongWarning": "Context above 400K may reduce quality. Around 300K is recommended.", "maxTokens": "Max Output Tokens", diff --git a/src/web-ui/src/locales/zh-CN/settings/models.json b/src/web-ui/src/locales/zh-CN/settings/models.json index 9df92914ff..f66e00fe25 100644 --- a/src/web-ui/src/locales/zh-CN/settings/models.json +++ b/src/web-ui/src/locales/zh-CN/settings/models.json @@ -393,6 +393,8 @@ "provider": "请求格式", "providerPlaceholder": "选择请求格式", "contextWindow": "上下文窗口大小", + "maxImagesPerRequest": "每次请求图片上限", + "maxImagesPerRequestHint": "可选的图片张数上限，包含历史消息和工具结果。留空保留现有行为。", "contextWindowHint": "用于判断何时压缩上下文，可小于模型实际支持的上限", "contextWindowLongWarning": "上下文超过 400K，可能影响效果，建议约 300K。", "maxTokens": "最大输出Tokens", diff --git a/src/web-ui/src/locales/zh-TW/settings/models.json b/src/web-ui/src/locales/zh-TW/settings/models.json index 903a9e5678..ef258c36a8 100644 --- a/src/web-ui/src/locales/zh-TW/settings/models.json +++ b/src/web-ui/src/locales/zh-TW/settings/models.json @@ -393,6 +393,8 @@ "provider": "請求格式", "providerPlaceholder": "選擇請求格式", "contextWindow": "上下文視窗大小", + "maxImagesPerRequest": "每次請求圖片上限", + "maxImagesPerRequestHint": "可選的圖片張數上限，包含歷史訊息和工具結果。留空保留現有行為。", "contextWindowHint": "用於判斷何時壓縮上下文，可小於模型實際支援的上限", "contextWindowLongWarning": "上下文超過 400K，可能影響效果，建議約 300K。", "maxTokens": "最大輸出Tokens", From 1bf6962785ec185119b4948e559414e72cde6ae0 Mon Sep 17 00:00:00 2001 From: uiharuayako <2451602772@qq.com> Date: Tue, 27 Oct 2026 14:33:17 +0800 Subject: [PATCH 2/2] fix: include translation dependency in image intake effect --- src/web-ui/src/flow_chat/components/ChatInput.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web-ui/src/flow_chat/components/ChatInput.tsx b/src/web-ui/src/flow_chat/components/ChatInput.tsx index 351975405d..05ac1f817c 100644 --- a/src/web-ui/src/flow_chat/components/ChatInput.tsx +++ b/src/web-ui/src/flow_chat/components/ChatInput.tsx @@ -3156,7 +3156,7 @@ export const ChatInput: React.FC = ({ return () => { globalEventBus.off('mcp-app:message', handleMcpAppMessage); }; - }, [addContext, clearPendingLargePastes, conversationScope, currentImageCount, dispatchInput, maxImageCount]); + }, [addContext, clearPendingLargePastes, conversationScope, currentImageCount, dispatchInput, maxImageCount, t]); React.useEffect(() => { const handleInsertContextTag = (event: Event) => {