From 756fd71c10f5ccae1808fefa5d934c6f728d1e66 Mon Sep 17 00:00:00 2001 From: kev1n77 Date: Mon, 26 Oct 2026 12:13:32 +0800 Subject: [PATCH 1/3] feat(media): add subscription image generation and model configuration --- .../flow-chat-presentation/src/registry.ts | 1 + src/apps/desktop/src/api/commands.rs | 37 + src/apps/desktop/src/lib.rs | 1 + src/crates/adapters/ai-adapters/AGENTS.md | 5 + src/crates/adapters/ai-adapters/Cargo.toml | 3 +- src/crates/adapters/ai-adapters/src/client.rs | 2 + .../adapters/ai-adapters/src/client/http.rs | 80 +- .../src/client/image_generation.rs | 1339 +++++++++++++++++ src/crates/assembly/core/AGENTS.md | 9 + src/crates/assembly/core/Cargo.toml | 2 + .../agentic/agents/definitions/modes/claw.rs | 1 + .../agents/definitions/modes/cowork.rs | 1 + .../agents/definitions/modes/deep_research.rs | 1 + .../definitions/subagents/general_purpose.rs | 1 + .../agents/definitions/subagents/swarm.rs | 1 + .../assembly/core/src/agentic/agents/mod.rs | 1 + .../core/src/agentic/image_generation/mod.rs | 595 ++++++++ src/crates/assembly/core/src/agentic/mod.rs | 2 + .../src/agentic/tools/agent-tool-exposure.md | 1 + .../implementations/generate_image_tool.rs | 488 ++++++ .../src/agentic/tools/implementations/mod.rs | 4 + .../tools/product_runtime/materialization.rs | 3 + .../core/src/agentic/tools/registry.rs | 1 + .../core/src/service/config/providers.rs | 211 ++- .../contracts/config-contracts/src/types.rs | 228 ++- src/crates/contracts/core-types/src/ai.rs | 55 + src/crates/contracts/core-types/src/lib.rs | 18 +- .../src/remote_surface/table.rs | 1 + .../execution/tool-provider-groups/src/lib.rs | 5 +- src/crates/services/services-core/Cargo.toml | 1 + src/crates/services/services-core/src/lib.rs | 2 + .../services-core/src/system_proxy.rs | 262 ++++ .../pages/ai/ModelPoolCardActions.tsx | 11 +- .../settings/pages/ai/ModelSettingsPage.tsx | 30 +- .../app/scenes/settings/settingsRegistry.ts | 9 +- .../tool-cards/builtinContextCardModel.ts | 17 + .../tool-cards/builtinToolCardModel.ts | 2 +- .../tool-cards/builtinToolCardPolicy.ts | 2 +- .../src/flow_chat/tool-cards/index.test.ts | 1 + src/web-ui/src/flow_chat/tool-cards/index.ts | 1 + .../api/service-api/MediaApi.ts | 61 + .../src/infrastructure/config/AGENTS.md | 9 + .../config/components/DefaultModelConfig.tsx | 4 + .../ImageGenerationEditor.tsx | 197 +++ .../ImageGenerationSection.scss | 39 + .../ImageGenerationSection.tsx | 90 ++ .../ImageGenerationSettings.test.tsx | 323 ++++ .../image-generation/imageDialects.test.ts | 45 + .../image-generation/imageDialects.ts | 130 ++ .../image-generation/imageModels.test.ts | 49 + .../image-generation/imageModels.ts | 79 + .../useImageGenerationSettings.ts | 173 +++ .../src/infrastructure/config/types/index.ts | 57 + .../i18n/presets/namespaceRegistry.ts | 1 + src/web-ui/src/locales/en-US/flow-chat.json | 3 + .../locales/en-US/settings/default-model.json | 2 +- .../src/locales/en-US/settings/media.json | 205 +++ .../src/locales/en-US/settings/models.json | 2 +- src/web-ui/src/locales/zh-CN/flow-chat.json | 3 + .../locales/zh-CN/settings/default-model.json | 2 +- .../src/locales/zh-CN/settings/media.json | 205 +++ .../src/locales/zh-CN/settings/models.json | 2 +- src/web-ui/src/locales/zh-TW/flow-chat.json | 3 + .../locales/zh-TW/settings/default-model.json | 2 +- .../src/locales/zh-TW/settings/media.json | 205 +++ .../src/locales/zh-TW/settings/models.json | 2 +- 66 files changed, 5279 insertions(+), 49 deletions(-) create mode 100644 src/crates/adapters/ai-adapters/src/client/image_generation.rs create mode 100644 src/crates/assembly/core/src/agentic/image_generation/mod.rs create mode 100644 src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs create mode 100644 src/crates/services/services-core/src/system_proxy.rs create mode 100644 src/web-ui/src/infrastructure/api/service-api/MediaApi.ts create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationEditor.tsx create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.tsx create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.test.ts create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/imageModels.test.ts create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/imageModels.ts create mode 100644 src/web-ui/src/infrastructure/config/components/image-generation/useImageGenerationSettings.ts create mode 100644 src/web-ui/src/locales/en-US/settings/media.json create mode 100644 src/web-ui/src/locales/zh-CN/settings/media.json create mode 100644 src/web-ui/src/locales/zh-TW/settings/media.json diff --git a/packages/flow-chat-presentation/src/registry.ts b/packages/flow-chat-presentation/src/registry.ts index 873cec6fde..694bd1dcf3 100644 --- a/packages/flow-chat-presentation/src/registry.ts +++ b/packages/flow-chat-presentation/src/registry.ts @@ -39,6 +39,7 @@ export const toolPresentationRegistry = { AgentList: { owner: 'standard', component: 'AgentRosterToolCard' }, SessionHistory: { owner: 'standard', component: 'SessionHistoryToolCard' }, analyze_image: { owner: 'standard', component: 'ImageAnalysisToolCard' }, + generate_image: { owner: 'standard', component: 'ImageAnalysisToolCard' }, GetTime: { owner: 'standard', component: 'TimeToolCard' }, ListMCPResources: { owner: 'standard', component: 'McpResourceToolCard' }, ReadMCPResource: { owner: 'standard', component: 'McpResourceToolCard' }, diff --git a/src/apps/desktop/src/api/commands.rs b/src/apps/desktop/src/api/commands.rs index 3912ab7d62..33ef013916 100644 --- a/src/apps/desktop/src/api/commands.rs +++ b/src/apps/desktop/src/api/commands.rs @@ -474,6 +474,14 @@ pub struct ListAIModelsByConfigRequest { pub config: openbitfun_core::service::config::types::AIModelConfig, } +#[derive(Debug, Deserialize)] +pub struct TestImageGenerationServiceRequest { + /// Draft service, not necessarily saved yet. + pub service: openbitfun_core::service::config::types::ImageServiceConfig, + /// Optional subject override for the check prompt. + pub prompt: Option, +} + #[derive(Debug, Deserialize)] pub struct UpdateAppStatusRequest { pub status: String, @@ -1238,6 +1246,35 @@ pub async fn test_ai_config_connection( } } +/// Runs one real generation against an image service draft so the settings page +/// can confirm the dialect, credentials and endpoint actually work together. +#[tauri::command] +pub async fn test_image_generation_service( + request: TestImageGenerationServiceRequest, +) -> Result { + const DEFAULT_PROMPT: &str = + "A simple red circle centered on a plain white background, flat vector style"; + + let prompt = request + .prompt + .as_deref() + .map(str::trim) + .filter(|prompt| !prompt.is_empty()) + .unwrap_or(DEFAULT_PROMPT); + let dialect = request.service.dialect; + let service_id = request.service.id.clone(); + + let outcome = + openbitfun_core::agentic::image_generation::test_service(&request.service, prompt).await; + + info!( + "Image generation service test completed: service={}, dialect={:?}, success={}, elapsed={}ms", + service_id, dialect, outcome.success, outcome.elapsed_ms + ); + + Ok(outcome) +} + #[tauri::command] pub async fn list_ai_models_by_config( state: State<'_, AppState>, diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index 265bcba307..93d5367206 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -1496,6 +1496,7 @@ pub async fn run() { get_statistics, test_ai_connection, test_ai_config_connection, + test_image_generation_service, list_ai_models_by_config, list_subscription_accounts, list_subscription_account_instances, diff --git a/src/crates/adapters/ai-adapters/AGENTS.md b/src/crates/adapters/ai-adapters/AGENTS.md index 94b7496a81..0e3f7639cf 100644 --- a/src/crates/adapters/ai-adapters/AGENTS.md +++ b/src/crates/adapters/ai-adapters/AGENTS.md @@ -276,6 +276,7 @@ cargo test -p openbitfun-ai-adapters cargo test -p openbitfun-ai-adapters --lib opencode_catalog cargo test --locked -p openbitfun-ai-adapters --lib providers::opencode::tests cargo test -p openbitfun-ai-adapters --lib models_dev::tests +cargo test -p openbitfun-ai-adapters --features image-generation --lib client::image_generation::tests cargo test -p openbitfun-ai-adapters --features subscription-auth subscription_auth cargo test --locked -p openbitfun-ai-adapters --features subscription-auth --lib subscription_auth::request_identity::tests:: cargo test --locked -p openbitfun-ai-adapters --features subscription-auth --lib client::sse::tests:: @@ -328,3 +329,7 @@ a query failure or inventing zero usage. The ignored to verify billing and the production quota read for that exact in-app slot. It may refresh credentials and quota cache, prints only response field names/status, and sends no inference request. This does not verify remote-host behavior. + +The image-generation filter uses synthetic HTTP credentials. The ignored Codex +probe requires a real in-app subscription login and generates a billed image; +run it explicitly only when live provider validation is needed. diff --git a/src/crates/adapters/ai-adapters/Cargo.toml b/src/crates/adapters/ai-adapters/Cargo.toml index 8432d49a5e..5cee82d8e5 100644 --- a/src/crates/adapters/ai-adapters/Cargo.toml +++ b/src/crates/adapters/ai-adapters/Cargo.toml @@ -24,7 +24,7 @@ async-trait = { workspace = true } base64 = { workspace = true, optional = true } openbitfun-agent-stream = { path = "../../execution/agent-stream" } openbitfun-core-types = { path = "../../contracts/core-types" } -openbitfun-services-core = { path = "../../services/services-core", features = ["tls-provider"] } +openbitfun-services-core = { path = "../../services/services-core", features = ["tls-provider", "system-proxy"] } chrono = { workspace = true } dirs = { workspace = true, optional = true } eventsource-stream = { workspace = true } @@ -52,6 +52,7 @@ keyring-core = { workspace = true, optional = true } zbus-secret-service-keyring-store = { workspace = true, optional = true } [features] +image-generation = ["dep:base64"] subscription-auth = [ "dep:base64", "openbitfun-services-core/credential-vault", diff --git a/src/crates/adapters/ai-adapters/src/client.rs b/src/crates/adapters/ai-adapters/src/client.rs index ab7feb01fe..0608c55e3e 100644 --- a/src/crates/adapters/ai-adapters/src/client.rs +++ b/src/crates/adapters/ai-adapters/src/client.rs @@ -7,6 +7,8 @@ pub(crate) mod format; pub(crate) mod healthcheck; pub(crate) mod http; +#[cfg(feature = "image-generation")] +pub mod image_generation; pub(crate) mod quirks; mod request_capacity; pub(crate) mod response_aggregator; diff --git a/src/crates/adapters/ai-adapters/src/client/http.rs b/src/crates/adapters/ai-adapters/src/client/http.rs index c9bec19bf0..318bc032af 100644 --- a/src/crates/adapters/ai-adapters/src/client/http.rs +++ b/src/crates/adapters/ai-adapters/src/client/http.rs @@ -2,7 +2,8 @@ use crate::client::AIClient; use crate::types::ProxyConfig; use anyhow::{anyhow, Result}; use log::{debug, error, info, warn}; -use reqwest::{Client, Proxy}; +use openbitfun_services_core::system_proxy::SystemProxy; +use reqwest::{Client, ClientBuilder, NoProxy, Proxy}; pub(crate) fn create_http_client( proxy_config: Option, @@ -30,40 +31,81 @@ pub(crate) fn create_http_client( ); } + builder = apply_proxy_policy(builder, proxy_config); + + match builder.build() { + Ok(client) => client, + Err(e) => { + error!( + "HTTP client initialization failed: {}, using default client", + e + ); + Client::new() + } + } +} + +/// Resolve the transport proxy policy in priority order. +/// +/// 1. An enabled proxy with an address is used as configured. +/// 2. A configured address with the switch off means "connect directly"; the +/// user turned the proxy off on purpose. +/// 3. No configured address falls back to the operating system proxy. Many users +/// reach providers only through a local proxy agent, and that setting lives in +/// the OS rather than in this app. +/// 4. Without any of the above the client keeps reqwest's default, which honours +/// the `HTTP_PROXY` / `HTTPS_PROXY` / `ALL_PROXY` environment variables. +fn apply_proxy_policy(builder: ClientBuilder, proxy_config: Option) -> ClientBuilder { if let Some(proxy_cfg) = proxy_config { - if proxy_cfg.enabled && !proxy_cfg.url.is_empty() { - match build_proxy(&proxy_cfg) { + let configured = !proxy_cfg.url.trim().is_empty(); + if proxy_cfg.enabled && configured { + return match build_proxy(&proxy_cfg) { Ok(proxy) => { info!("Using proxy: {}", proxy_cfg.url); - builder = builder.proxy(proxy); + builder.proxy(proxy) } Err(e) => { error!( "Proxy configuration failed: {}, proceeding without proxy", e ); - builder = builder.no_proxy(); + builder.no_proxy() } - } - } else { - builder = builder.no_proxy(); + }; + } + if configured { + info!("Proxy disabled by configuration, connecting directly"); + return builder.no_proxy(); } - } else { - builder = builder.no_proxy(); } - match builder.build() { - Ok(client) => client, - Err(e) => { - error!( - "HTTP client initialization failed: {}, using default client", - e - ); - Client::new() - } + match openbitfun_services_core::system_proxy::detect() { + Some(system) => match system_proxy(&system) { + Some(proxy) => { + info!("Using operating system proxy: {}", system.url); + builder.proxy(proxy) + } + None => { + warn!( + "Operating system proxy {} could not be applied, connecting directly", + system.url + ); + builder.no_proxy() + } + }, + None => builder, } } +/// Build a reqwest proxy from an operating system setting. +/// +/// Loopback targets always bypass the proxy so local model servers keep working. +fn system_proxy(system: &SystemProxy) -> Option { + let proxy = Proxy::all(&system.url).ok()?; + let no_proxy = system.no_proxy.as_deref().and_then(NoProxy::from_string); + Some(proxy.no_proxy(no_proxy)) +} + pub(crate) fn build_proxy(config: &ProxyConfig) -> Result { let proxy_url = normalize_proxy_url(&config.url); let mut proxy = Proxy::all(&proxy_url).map_err(|e| anyhow!("Failed to create proxy: {}", e))?; diff --git a/src/crates/adapters/ai-adapters/src/client/image_generation.rs b/src/crates/adapters/ai-adapters/src/client/image_generation.rs new file mode 100644 index 0000000000..af94910866 --- /dev/null +++ b/src/crates/adapters/ai-adapters/src/client/image_generation.rs @@ -0,0 +1,1339 @@ +//! Image generation over an explicit protocol dialect. +//! +//! The dialect alone decides the endpoint, request body shape, size syntax and +//! response parsing, so a new provider is a new dialect arm rather than a new +//! transport. This module deliberately does not know the persisted config shape: +//! callers resolve one configured service into [`ImageGenerationTarget`]. +//! +//! Results always come back as decoded bytes. Several providers hand back +//! short-lived URLs (DashScope: 24 hours), so persisting them is the caller's job. + +use crate::client::AIClient; +use crate::types::ProxyConfig; +use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; +use base64::Engine as _; +use log::debug; +use openbitfun_core_types::ImageDialect; +use reqwest::Client; +use serde_json::{json, Map, Value}; +use std::collections::HashMap; +use std::time::Duration; + +/// Whole-request budget. Image generation is slow but not unbounded. +const REQUEST_TIMEOUT_SECS: u64 = 180; +/// Guard against a provider streaming an unbounded body. +const MAX_BODY_BYTES: usize = 64 * 1024 * 1024; +const DEFAULT_MIME: &str = "image/png"; + +/// Substrings that mean the provider refused on safety grounds rather than +/// failing. Reported separately so the UI can say "blocked" instead of "failed". +const CONTENT_FILTER_MARKERS: &[&str] = &[ + "content_policy", + "content policy", + "content_filter", + "moderation_blocked", + "safety system", + "datainspectionfailed", + "inappropriatecontent", + "sensitivecontent", + "risk control", +]; + +/// Protocol-level target resolved by the caller from one configured service. +pub struct ImageGenerationTarget<'a> { + pub dialect: ImageDialect, + pub base_url: &'a str, + /// Full URL override. When set it replaces `base_url` + dialect path. + pub request_url: Option<&'a str>, + pub model: &'a str, + pub api_key: &'a str, + pub custom_headers: Option<&'a HashMap>, + /// `Some("replace")` sends only the custom headers. + pub custom_headers_mode: Option<&'a str>, + pub skip_ssl_verify: bool, + pub proxy: Option, +} + +/// One generation call. Unset fields fall back to the provider default. +#[derive(Debug, Clone, Default)] +pub struct ImageGenerationRequest { + pub prompt: String, + pub negative_prompt: Option, + pub size: Option, + pub n: Option, + pub quality: Option, + pub background: Option, + pub output_format: Option, + pub watermark: Option, + pub prompt_extend: Option, + pub seed: Option, +} + +/// One decoded image. +#[derive(Debug)] +pub struct GeneratedImage { + pub bytes: Vec, + pub mime: String, +} + +impl GeneratedImage { + /// Standard base64 payload for a settings-page preview. Kept on the adapter + /// so UI hosts do not need to depend on an encoder. + pub fn base64(&self) -> String { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.encode(&self.bytes) + } +} + +#[derive(Debug)] +pub struct ImageGenerationResponse { + pub images: Vec, +} + +/// Why a generation failed. Kept coarse and stable: the settings surface maps +/// these to distinct guidance instead of showing one opaque error. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ImageGenerationErrorKind { + /// The configured service is unusable (missing endpoint, empty prompt). + Config, + /// Rejected credentials. + Auth, + /// Rate limited or quota exhausted. + RateLimited, + /// Rejected request, most often a dialect or parameter mismatch. + InvalidRequest, + /// Refused by the provider's safety system. + ContentFiltered, + Timeout, + /// Network layer failure. + Transport, + /// Provider-side failure. + Provider, + /// The provider answered successfully but produced no image. + EmptyResult, +} + +impl ImageGenerationErrorKind { + /// Stable wire name consumed by the settings surface. Adding a variant is a + /// frontend-visible change, so keep these names fixed. + pub fn wire_name(self) -> &'static str { + match self { + Self::Config => "config", + Self::Auth => "auth", + Self::RateLimited => "rate_limited", + Self::InvalidRequest => "invalid_request", + Self::ContentFiltered => "content_filtered", + Self::Timeout => "timeout", + Self::Transport => "transport", + Self::Provider => "provider", + Self::EmptyResult => "empty_result", + } + } +} + +#[derive(Debug)] +pub struct ImageGenerationError { + pub kind: ImageGenerationErrorKind, + pub message: String, +} + +impl ImageGenerationError { + fn new(kind: ImageGenerationErrorKind, message: impl Into) -> Self { + Self { + kind, + message: message.into(), + } + } + + fn config(message: impl Into) -> Self { + Self::new(ImageGenerationErrorKind::Config, message) + } + + fn empty_result(message: impl Into) -> Self { + Self::new(ImageGenerationErrorKind::EmptyResult, message) + } +} + +impl std::fmt::Display for ImageGenerationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message) + } +} + +impl std::error::Error for ImageGenerationError {} + +/// Resolve the absolute request URL for a target. +/// +/// A user may paste either the service root or the full endpoint into the base +/// URL field, so an already-terminated path is not appended twice. +pub(crate) fn endpoint(target: &ImageGenerationTarget<'_>) -> Result { + if let Some(url) = target + .request_url + .map(str::trim) + .filter(|url| !url.is_empty()) + { + return Ok(url.to_string()); + } + + let base = target.base_url.trim().trim_end_matches('/'); + if base.is_empty() { + return Err(ImageGenerationError::config( + "image service has no request url and no base url", + )); + } + + let path = target.dialect.endpoint_path(); + if base.ends_with(path) { + Ok(base.to_string()) + } else { + Ok(format!("{base}{path}")) + } +} + +/// Rewrite a size expression to the requested separator. +/// +/// `size` stays a dialect-native string in config, so a user who typed the other +/// dialect's separator still gets a valid request instead of a 400. +fn normalize_size(raw: &str, separator: char) -> String { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return trimmed.to_string(); + } + trimmed + .replace(['x', 'X', '×', '*'], "\u{0}") + .replace('\u{0}', &separator.to_string()) +} + +fn clamped_n(request: &ImageGenerationRequest, dialect: ImageDialect) -> u32 { + request + .n + .unwrap_or(1) + .clamp(1, dialect.max_images_per_request()) +} + +/// OpenAI Images body. `response_format` is deliberately omitted: the GPT image +/// models return base64 unconditionally and reject the field, while compatible +/// providers may answer with a URL. Both shapes are parsed. +fn openai_body(target: &ImageGenerationTarget<'_>, request: &ImageGenerationRequest) -> Value { + let mut body = json!({ + "model": target.model, + "prompt": request.prompt, + "n": clamped_n(request, target.dialect), + }); + if let Some(size) = request.size.as_deref() { + body["size"] = json!(normalize_size(size, 'x')); + } + if let Some(quality) = request.quality.as_deref() { + body["quality"] = json!(quality); + } + if let Some(background) = request.background.as_deref() { + body["background"] = json!(background); + } + if let Some(output_format) = request.output_format.as_deref() { + body["output_format"] = json!(output_format); + } + body +} + +/// DashScope native Qwen-Image body. +fn dashscope_body(target: &ImageGenerationTarget<'_>, request: &ImageGenerationRequest) -> Value { + let mut parameters = Map::new(); + if let Some(size) = request.size.as_deref() { + parameters.insert("size".to_string(), json!(normalize_size(size, '*'))); + } + parameters.insert("n".to_string(), json!(clamped_n(request, target.dialect))); + if let Some(negative_prompt) = request.negative_prompt.as_deref() { + parameters.insert("negative_prompt".to_string(), json!(negative_prompt)); + } + if let Some(watermark) = request.watermark { + parameters.insert("watermark".to_string(), json!(watermark)); + } + if let Some(prompt_extend) = request.prompt_extend { + parameters.insert("prompt_extend".to_string(), json!(prompt_extend)); + } + if let Some(seed) = request.seed { + parameters.insert("seed".to_string(), json!(seed)); + } + + json!({ + "model": target.model, + "input": { + "messages": [{ + "role": "user", + "content": [{ "text": request.prompt }], + }], + }, + "parameters": Value::Object(parameters), + }) +} + +fn build_body(target: &ImageGenerationTarget<'_>, request: &ImageGenerationRequest) -> Value { + match target.dialect { + ImageDialect::OpenAiImages => openai_body(target, request), + ImageDialect::DashScopeNative => dashscope_body(target, request), + ImageDialect::ChatgptCodex => chatgpt_codex_body(target.model, request), + } +} + +/// ChatGPT subscription body. +/// +/// The image is not requested from an image endpoint: the model is asked to call +/// its built-in `image_generation` tool, and the picture arrives as a Responses +/// output item. `store: false` is required by the Codex backend, and `stream` +/// is what produces the SSE events the caller parses. +fn chatgpt_codex_body(model: &str, request: &ImageGenerationRequest) -> Value { + let mut tool = json!({ "type": "image_generation" }); + // The tool accepts the same knobs as the image API, but only when set. + if let Some(size) = request.size.as_deref() { + tool["size"] = json!(normalize_size(size, 'x')); + } + if let Some(quality) = request.quality.as_deref() { + tool["quality"] = json!(quality); + } + if let Some(background) = request.background.as_deref() { + tool["background"] = json!(background); + } + + json!({ + "model": model, + "stream": true, + "store": false, + "tools": [tool], + "input": [{ + "role": "user", + "content": [{ "type": "input_text", "text": request.prompt }], + }], + }) +} + +/// Where a parsed image still has to come from. +#[derive(Debug)] +enum ImageSource { + Inline { bytes: Vec, mime: String }, + Remote(String), +} + +fn parse_openai_images(value: &Value) -> Result, ImageGenerationError> { + let Some(data) = value.get("data").and_then(Value::as_array) else { + return Err(ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + "response is missing the 'data' array", + )); + }; + + let mut sources = Vec::new(); + for item in data { + let inline = item + .get("b64_json") + .and_then(Value::as_str) + .or_else(|| item.get("base64").and_then(Value::as_str)); + if let Some(encoded) = inline { + let bytes = BASE64_STANDARD.decode(encoded.trim()).map_err(|err| { + ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + format!("image payload is not valid base64: {err}"), + ) + })?; + let mime = sniff_mime(&bytes).unwrap_or(DEFAULT_MIME).to_string(); + sources.push(ImageSource::Inline { bytes, mime }); + continue; + } + + let remote = item + .get("url") + .and_then(Value::as_str) + .or_else(|| item.get("image_url").and_then(Value::as_str)); + if let Some(url) = remote { + sources.push(ImageSource::Remote(url.to_string())); + } + } + + Ok(sources) +} + +fn parse_dashscope_images(value: &Value) -> Result, ImageGenerationError> { + let mut sources = Vec::new(); + + // Synchronous multimodal generation: output.choices[].message.content[].image + if let Some(choices) = value.pointer("/output/choices").and_then(Value::as_array) { + for choice in choices { + let Some(content) = choice.pointer("/message/content").and_then(Value::as_array) else { + continue; + }; + for part in content { + if let Some(image) = part.get("image").and_then(Value::as_str) { + sources.push(ImageSource::Remote(image.to_string())); + } + } + } + } + + // Task-shaped results: output.results[].url + if let Some(results) = value.pointer("/output/results").and_then(Value::as_array) { + for result in results { + if let Some(url) = result.get("url").and_then(Value::as_str) { + sources.push(ImageSource::Remote(url.to_string())); + } + } + } + + Ok(sources) +} + +fn parse_images( + dialect: ImageDialect, + value: &Value, +) -> Result, ImageGenerationError> { + match dialect { + ImageDialect::OpenAiImages => parse_openai_images(value), + ImageDialect::DashScopeNative => parse_dashscope_images(value), + // Streamed by `parse_chatgpt_codex_stream`; reaching here means the JSON + // path was taken for a dialect that never answers with one document. + ImageDialect::ChatgptCodex => Err(ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + "the ChatGPT subscription dialect streams its result and cannot be read as JSON", + )), + } +} + +/// Reads the Responses SSE transcript and returns the image the model produced. +/// +/// The backend emits progress events while the tool runs, so only the terminal +/// `output_item.done` carries `result`. A response that completes without an +/// image item means the model declined to call the tool, which is a different +/// failure from an empty payload and is reported as such. +fn parse_chatgpt_codex_stream(raw: &[u8]) -> Result, ImageGenerationError> { + let text = String::from_utf8_lossy(raw); + let mut sources = Vec::new(); + let mut failure: Option<(Option, String)> = None; + let mut completed_without_image = false; + + for line in text.lines() { + let Some(payload) = line.strip_prefix("data:") else { + continue; + }; + let payload = payload.trim(); + if payload.is_empty() || payload == "[DONE]" { + continue; + } + let Ok(value) = serde_json::from_str::(payload) else { + continue; + }; + match value + .get("type") + .and_then(Value::as_str) + .unwrap_or_default() + { + "response.output_item.done" => { + let item = value.get("item"); + if item + .and_then(|item| item.get("type")) + .and_then(Value::as_str) + != Some("image_generation_call") + { + continue; + } + let Some(encoded) = item + .and_then(|item| item.get("result")) + .and_then(Value::as_str) + else { + continue; + }; + let bytes = BASE64_STANDARD.decode(encoded.trim()).map_err(|err| { + ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + format!("image payload is not valid base64: {err}"), + ) + })?; + let mime = sniff_mime(&bytes).unwrap_or(DEFAULT_MIME).to_string(); + sources.push(ImageSource::Inline { bytes, mime }); + } + "response.failed" => { + let error = value.pointer("/response/error"); + failure = Some(( + error + .and_then(|error| error.get("code")) + .and_then(Value::as_str) + .map(str::to_string), + error + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .unwrap_or("the model reported a failed response") + .to_string(), + )); + } + "error" => { + let error = value.get("error"); + failure = Some(( + error + .and_then(|error| error.get("code")) + .and_then(Value::as_str) + .map(str::to_string), + error + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .unwrap_or("the provider streamed an error") + .to_string(), + )); + } + "response.completed" => completed_without_image = true, + _ => {} + } + } + + if let Some((code, message)) = failure { + return Err(ImageGenerationError::new( + codex_stream_error_kind(&message), + match code { + Some(code) => format!("provider failed the image request ({code}): {message}"), + None => format!("provider failed the image request: {message}"), + }, + )); + } + if sources.is_empty() { + return Err(ImageGenerationError::empty_result( + if completed_without_image { + "the response completed without calling the image tool; pick a model that can generate images" + } else { + "the provider streamed no image result" + }, + )); + } + Ok(sources) +} + +/// Map a streamed error message onto the guidance class the UI renders. +fn codex_stream_error_kind(message: &str) -> ImageGenerationErrorKind { + let lowered = message.to_ascii_lowercase(); + if looks_like_content_filter(message) { + ImageGenerationErrorKind::ContentFiltered + } else if lowered.contains("rate limit") || lowered.contains("usage limit") { + ImageGenerationErrorKind::RateLimited + } else if lowered.contains("timed out") || lowered.contains("timeout") { + ImageGenerationErrorKind::Timeout + } else { + ImageGenerationErrorKind::Provider + } +} + +/// Error body of a rejected subscription request. +/// +/// The Codex backend answers JSON (`{"detail": ...}` for request problems, +/// `{"error": {...}}` for API errors) even though a successful call streams. +fn chatgpt_codex_error(status: reqwest::StatusCode, raw: &[u8]) -> ImageGenerationError { + let value: Value = serde_json::from_slice(raw).unwrap_or(Value::Null); + if let Some(detail) = value.get("detail").and_then(Value::as_str) { + return ImageGenerationError::new( + classify_status(status), + format!("provider rejected the request: {detail}"), + ); + } + if !value.is_null() { + return error_from_response(status, &value); + } + ImageGenerationError::new( + classify_status(status), + format!( + "provider rejected the request (HTTP {status}): {}", + String::from_utf8_lossy(&raw[..raw.len().min(200)]) + ), + ) +} + +/// Ask the account's own catalog which model to drive the image tool with. +async fn discover_chatgpt_codex_model( + client: &Client, + target: &ImageGenerationTarget<'_>, +) -> Result { + let base = match target.request_url { + Some(request_url) => request_url + .trim_end_matches('/') + .trim_end_matches("/responses") + .to_string(), + None => target.base_url.trim_end_matches('/').to_string(), + }; + let url = format!("{base}/models?client_version=1.0.0"); + + let mut builder = client + .get(&url) + .bearer_auth(target.api_key.trim()) + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS)); + if let Some(headers) = target.custom_headers { + for (key, value) in headers { + builder = builder.header(key.as_str(), value.as_str()); + } + } + let response = builder.send().await.map_err(classify_transport)?; + let status = response.status(); + let raw = response.bytes().await.map_err(classify_transport)?; + let value: Value = serde_json::from_slice(&raw).map_err(|err| { + ImageGenerationError::new( + classify_status(status), + format!("could not read the subscription model catalog: {err}"), + ) + })?; + if !status.is_success() { + return Err(error_from_response(status, &value)); + } + + let model = value + .get("models") + .and_then(Value::as_array) + .and_then(|models| { + let mut entries: Vec<(u64, String)> = models + .iter() + .filter_map(|model| { + let slug = model.get("slug").and_then(Value::as_str)?.trim(); + if slug.is_empty() { + return None; + } + let priority = model + .get("priority") + .and_then(Value::as_u64) + .unwrap_or(10_000); + Some((priority, slug.to_string())) + }) + .collect(); + entries.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + entries.into_iter().next().map(|(_, slug)| slug) + }); + + model.ok_or_else(|| { + ImageGenerationError::config( + "the subscription login exposes no model to drive image generation; set a model name explicitly", + ) + }) +} + +/// Extract a provider error from the shapes seen in the wild: +/// `{error:{code,message}}`, `{code,message}` and DashScope's `{output:{code,message}}`. +fn extract_error(value: &Value) -> Option<(Option, String)> { + if let Some(error) = value.get("error") { + if let Some(message) = error.get("message").and_then(Value::as_str) { + return Some(( + error + .get("code") + .and_then(Value::as_str) + .map(str::to_string), + message.to_string(), + )); + } + if let Some(message) = error.as_str() { + return Some((None, message.to_string())); + } + } + + if let Some(message) = value.get("message").and_then(Value::as_str) { + return Some(( + value + .get("code") + .and_then(Value::as_str) + .map(str::to_string), + message.to_string(), + )); + } + + if let Some(output) = value.get("output") { + if let Some(message) = output.get("message").and_then(Value::as_str) { + return Some(( + output + .get("code") + .and_then(Value::as_str) + .map(str::to_string), + message.to_string(), + )); + } + } + + None +} + +fn looks_like_content_filter(message: &str) -> bool { + let lowered = message.to_lowercase(); + CONTENT_FILTER_MARKERS + .iter() + .any(|marker| lowered.contains(marker)) +} + +/// DashScope reports an asynchronous failure inside a 200 response. +fn dialect_reported_failure(dialect: ImageDialect, value: &Value) -> Option { + if dialect != ImageDialect::DashScopeNative { + return None; + } + let status = value + .pointer("/output/task_status") + .and_then(Value::as_str)?; + matches!(status, "FAILED" | "CANCELED" | "UNKNOWN").then(|| status.to_string()) +} + +fn classify_status(status: reqwest::StatusCode) -> ImageGenerationErrorKind { + match status.as_u16() { + 401 | 403 => ImageGenerationErrorKind::Auth, + 429 => ImageGenerationErrorKind::RateLimited, + 408 | 504 => ImageGenerationErrorKind::Timeout, + 400 | 404 | 405 | 409 | 413 | 415 | 422 => ImageGenerationErrorKind::InvalidRequest, + _ => ImageGenerationErrorKind::Provider, + } +} + +fn error_from_response(status: reqwest::StatusCode, value: &Value) -> ImageGenerationError { + let kind = classify_status(status); + let (code, message) = match extract_error(value) { + Some((code, message)) => (code, message), + None => (None, value.to_string()), + }; + let message = match code { + Some(code) => format!("provider rejected the request ({code}): {message}"), + None => format!("provider rejected the request: {message}"), + }; + let kind = if looks_like_content_filter(&message) { + ImageGenerationErrorKind::ContentFiltered + } else { + kind + }; + ImageGenerationError::new(kind, message) +} + +fn classify_transport(err: reqwest::Error) -> ImageGenerationError { + let kind = if err.is_timeout() { + ImageGenerationErrorKind::Timeout + } else { + ImageGenerationErrorKind::Transport + }; + // A timeout message that names the budget lets the reader tell "slow + // provider" apart from "unreachable endpoint" without a rebuild. + let budget = if err.is_timeout() { + let seconds = if err.is_connect() { + AIClient::STREAM_CONNECT_TIMEOUT_SECS + } else { + REQUEST_TIMEOUT_SECS + }; + format!(" after waiting up to {seconds}s") + } else { + String::new() + }; + ImageGenerationError::new( + kind, + format!( + "image generation request failed while {}{budget}: {err}", + transport_stage(&err) + ), + ) +} + +/// Name the step that failed so a connection problem is diagnosable. +/// +/// reqwest only exposes a flat error, so the stage is read from the source text. +/// Callers see the same kind either way; this only improves the message. +fn transport_stage(err: &reqwest::Error) -> &'static str { + let text = err.to_string().to_lowercase(); + if text.contains("dns error") || text.contains("failed to lookup address") { + "resolving the host name" + } else if text.contains("certificate") + || text.contains("tls") + || text.contains("handshake") + || text.contains("invalidcertificate") + { + "negotiating TLS" + } else if text.contains("connection refused") || text.contains("connection reset") { + "connecting to the host" + } else if text.contains("timeout") || text.contains("timed out") { + "waiting for the response" + } else { + "sending the request" + } +} + +fn sniff_mime(bytes: &[u8]) -> Option<&'static str> { + if bytes.starts_with(&[0x89, b'P', b'N', b'G']) { + return Some("image/png"); + } + if bytes.starts_with(&[0xFF, 0xD8, 0xFF]) { + return Some("image/jpeg"); + } + if bytes.len() >= 12 && &bytes[0..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + return Some("image/webp"); + } + if bytes.starts_with(b"GIF8") { + return Some("image/gif"); + } + None +} + +async fn download_image( + client: &Client, + url: &str, +) -> Result<(Vec, String), ImageGenerationError> { + let response = client + .get(url) + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS)) + .send() + .await + .map_err(classify_transport)?; + + let status = response.status(); + let header_mime = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(|value| { + value + .split(';') + .next() + .unwrap_or(DEFAULT_MIME) + .trim() + .to_string() + }) + .filter(|value| value.starts_with("image/")); + + let bytes = response.bytes().await.map_err(classify_transport)?; + if !status.is_success() { + return Err(ImageGenerationError::new( + classify_status(status), + format!("failed to download the generated image: HTTP {status}"), + )); + } + if bytes.len() > MAX_BODY_BYTES { + return Err(ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + "generated image exceeds the size limit", + )); + } + + let mime = header_mime + .or_else(|| sniff_mime(&bytes).map(str::to_string)) + .unwrap_or_else(|| DEFAULT_MIME.to_string()); + Ok((bytes.to_vec(), mime)) +} + +/// Generate images for one configured service. +pub async fn generate( + target: &ImageGenerationTarget<'_>, + request: ImageGenerationRequest, +) -> Result { + if request.prompt.trim().is_empty() { + return Err(ImageGenerationError::config("prompt must not be empty")); + } + // The subscription route asks a chat model to call its image tool, so the + // model is whatever the account's catalog offers. Discover it when the + // service left the field empty instead of guessing a slug. + let discover_model = + target.dialect == ImageDialect::ChatgptCodex && target.model.trim().is_empty(); + if target.model.trim().is_empty() && !discover_model { + return Err(ImageGenerationError::config("model name must not be empty")); + } + + let url = endpoint(target)?; + let client = + crate::client::http::create_http_client(target.proxy.clone(), target.skip_ssl_verify); + + let model = if discover_model { + discover_chatgpt_codex_model(&client, target).await? + } else { + target.model.trim().to_string() + }; + let body = if target.dialect == ImageDialect::ChatgptCodex { + chatgpt_codex_body(&model, &request) + } else { + build_body(target, &request) + }; + + debug!( + "Image generation request: dialect={:?} model={} url={}", + target.dialect, model, url + ); + + let payload = serde_json::to_vec(&body).map_err(|err| { + ImageGenerationError::config(format!("failed to encode the request body: {err}")) + })?; + + let has_custom_headers = target + .custom_headers + .is_some_and(|headers| !headers.is_empty()); + let merge_headers = target.custom_headers_mode.as_deref() != Some("replace"); + + let mut builder = client + .post(&url) + .timeout(Duration::from_secs(REQUEST_TIMEOUT_SECS)) + .body(payload); + + if !has_custom_headers || merge_headers { + builder = builder + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header( + reqwest::header::AUTHORIZATION, + format!("Bearer {}", target.api_key.trim()), + ); + } + if let Some(headers) = target.custom_headers { + for (key, value) in headers { + builder = builder.header(key.as_str(), value.as_str()); + } + } + + let response = builder.send().await.map_err(classify_transport)?; + let status = response.status(); + let raw = response.bytes().await.map_err(classify_transport)?; + if raw.len() > MAX_BODY_BYTES { + return Err(ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + "image generation response exceeds the size limit", + )); + } + + // The subscription route answers with an SSE transcript, not a JSON document. + if target.dialect == ImageDialect::ChatgptCodex { + if !status.is_success() { + return Err(chatgpt_codex_error(status, &raw)); + } + let sources = parse_chatgpt_codex_stream(&raw)?; + let mut images = Vec::with_capacity(sources.len()); + for source in sources { + match source { + ImageSource::Inline { bytes, mime } => images.push(GeneratedImage { bytes, mime }), + ImageSource::Remote(url) => { + let (bytes, mime) = download_image(&client, &url).await?; + images.push(GeneratedImage { bytes, mime }); + } + } + } + return Ok(ImageGenerationResponse { images }); + } + + let value: Value = serde_json::from_slice(&raw).map_err(|err| { + ImageGenerationError::new( + classify_status(status), + format!( + "provider returned a non-JSON response (HTTP {status}): {err}; body starts with {:?}", + String::from_utf8_lossy(&raw[..raw.len().min(120)]) + ), + ) + })?; + + if !status.is_success() { + return Err(error_from_response(status, &value)); + } + + if let Some(task_status) = dialect_reported_failure(target.dialect, &value) { + let (code, message) = extract_error(&value) + .map(|(code, message)| (code, message)) + .unwrap_or((None, format!("task finished with status {task_status}"))); + return Err(ImageGenerationError::new( + ImageGenerationErrorKind::Provider, + match code { + Some(code) => format!("image task failed ({code}): {message}"), + None => format!("image task failed: {message}"), + }, + )); + } + + if let Some((code, message)) = extract_error(&value) { + // Some compatible providers answer 200 with an error envelope. + let kind = if looks_like_content_filter(&message) { + ImageGenerationErrorKind::ContentFiltered + } else { + ImageGenerationErrorKind::Provider + }; + return Err(ImageGenerationError::new( + kind, + match code { + Some(code) => format!("provider returned an error ({code}): {message}"), + None => format!("provider returned an error: {message}"), + }, + )); + } + + let sources = parse_images(target.dialect, &value)?; + if sources.is_empty() { + return Err(ImageGenerationError::empty_result( + "provider returned no images", + )); + } + + let mut images = Vec::with_capacity(sources.len()); + for source in sources { + match source { + ImageSource::Inline { bytes, mime } => images.push(GeneratedImage { bytes, mime }), + ImageSource::Remote(url) => { + let (bytes, mime) = download_image(&client, &url).await?; + images.push(GeneratedImage { bytes, mime }); + } + } + } + + debug!( + "Image generation finished: dialect={:?} images={}", + target.dialect, + images.len() + ); + + Ok(ImageGenerationResponse { images }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn target<'a>(dialect: ImageDialect, base_url: &'a str) -> ImageGenerationTarget<'a> { + ImageGenerationTarget { + dialect, + base_url, + request_url: None, + model: "test-model", + api_key: "sk-test", + custom_headers: None, + custom_headers_mode: None, + skip_ssl_verify: false, + proxy: None, + } + } + + #[test] + fn endpoint_appends_the_dialect_path() { + assert_eq!( + endpoint(&target( + ImageDialect::OpenAiImages, + "https://api.openai.com/v1" + )) + .unwrap(), + "https://api.openai.com/v1/images/generations" + ); + assert_eq!( + endpoint(&target( + ImageDialect::DashScopeNative, + "https://dashscope.aliyuncs.com/api/v1/" + )) + .unwrap(), + "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + ); + } + + #[test] + fn endpoint_does_not_append_the_path_twice() { + let url = "https://api.openai.com/v1/images/generations"; + assert_eq!( + endpoint(&target(ImageDialect::OpenAiImages, url)).unwrap(), + url + ); + } + + #[test] + fn request_url_override_wins() { + let mut service = target(ImageDialect::OpenAiImages, "https://api.openai.com/v1"); + service.request_url = Some("https://proxy.example.com/custom/path"); + assert_eq!( + endpoint(&service).unwrap(), + "https://proxy.example.com/custom/path" + ); + } + + #[test] + fn endpoint_requires_some_url() { + let error = endpoint(&target(ImageDialect::OpenAiImages, " ")).unwrap_err(); + assert_eq!(error.kind, ImageGenerationErrorKind::Config); + } + + #[test] + fn size_separator_follows_the_dialect() { + assert_eq!(normalize_size("1024x1024", 'x'), "1024x1024"); + assert_eq!(normalize_size("1024*1024", 'x'), "1024x1024"); + assert_eq!(normalize_size("2048*2048", '*'), "2048*2048"); + assert_eq!(normalize_size("2048x1152", '*'), "2048*1152"); + } + + #[test] + fn openai_body_omits_absent_optional_fields() { + let body = openai_body( + &target(ImageDialect::OpenAiImages, "https://api.openai.com/v1"), + &ImageGenerationRequest { + prompt: "a red fox".to_string(), + ..ImageGenerationRequest::default() + }, + ); + + assert_eq!(body["model"], "test-model"); + assert_eq!(body["prompt"], "a red fox"); + assert_eq!(body["n"], 1); + assert!(body.get("size").is_none()); + assert!(body.get("quality").is_none()); + assert!(body.get("response_format").is_none()); + } + + #[test] + fn openai_body_uses_x_separator_and_clamps_n() { + let body = openai_body( + &target(ImageDialect::OpenAiImages, "https://api.openai.com/v1"), + &ImageGenerationRequest { + prompt: "a red fox".to_string(), + size: Some("1024*1536".to_string()), + n: Some(99), + quality: Some("high".to_string()), + ..ImageGenerationRequest::default() + }, + ); + + assert_eq!(body["size"], "1024x1536"); + assert_eq!(body["n"], 10); + assert_eq!(body["quality"], "high"); + } + + #[test] + fn dashscope_body_nests_prompt_and_uses_star_separator() { + let body = dashscope_body( + &target( + ImageDialect::DashScopeNative, + "https://dashscope.aliyuncs.com/api/v1", + ), + &ImageGenerationRequest { + prompt: "a red fox".to_string(), + size: Some("2048x1152".to_string()), + n: Some(99), + negative_prompt: Some("blur".to_string()), + watermark: Some(false), + prompt_extend: Some(true), + seed: Some(42), + ..ImageGenerationRequest::default() + }, + ); + + assert_eq!(body["input"]["messages"][0]["role"], "user"); + assert_eq!( + body["input"]["messages"][0]["content"][0]["text"], + "a red fox" + ); + assert_eq!(body["parameters"]["size"], "2048*1152"); + assert_eq!(body["parameters"]["n"], 6); + assert_eq!(body["parameters"]["negative_prompt"], "blur"); + assert_eq!(body["parameters"]["watermark"], false); + assert_eq!(body["parameters"]["prompt_extend"], true); + assert_eq!(body["parameters"]["seed"], 42); + } + + #[test] + fn openai_parser_reads_inline_and_remote_sources() { + let inline = BASE64_STANDARD.encode([0x89, b'P', b'N', b'G', 1, 2, 3]); + let value = json!({ + "data": [ + { "b64_json": inline }, + { "url": "https://example.com/a.png" } + ] + }); + + let sources = parse_openai_images(&value).unwrap(); + + assert_eq!(sources.len(), 2); + match &sources[0] { + ImageSource::Inline { bytes, mime } => { + assert_eq!(mime, "image/png"); + assert_eq!(bytes.len(), 7); + } + ImageSource::Remote(_) => panic!("expected an inline image"), + } + match &sources[1] { + ImageSource::Remote(url) => assert_eq!(url, "https://example.com/a.png"), + ImageSource::Inline { .. } => panic!("expected a remote image"), + } + } + + #[test] + fn openai_parser_rejects_a_payload_without_data() { + let error = parse_openai_images(&json!({"unexpected": true})).unwrap_err(); + assert_eq!(error.kind, ImageGenerationErrorKind::Provider); + } + + #[test] + fn dashscope_parser_reads_nested_content_images() { + let value = json!({ + "output": { + "task_status": "SUCCEEDED", + "choices": [{ + "message": { + "content": [ + { "image": "https://example.com/a.png" }, + { "image": "https://example.com/b.png" } + ] + } + }] + } + }); + + let sources = parse_dashscope_images(&value).unwrap(); + + assert_eq!(sources.len(), 2); + match &sources[1] { + ImageSource::Remote(url) => assert_eq!(url, "https://example.com/b.png"), + ImageSource::Inline { .. } => panic!("expected a remote image"), + } + } + + #[test] + fn dashscope_parser_reads_task_results() { + let value = json!({"output": {"results": [{"url": "https://example.com/a.png"}]}}); + + assert_eq!(parse_dashscope_images(&value).unwrap().len(), 1); + } + + #[test] + fn error_extraction_covers_the_common_envelopes() { + assert_eq!( + extract_error(&json!({"error": {"code": "bad", "message": "nope"}})).unwrap(), + (Some("bad".to_string()), "nope".to_string()) + ); + assert_eq!( + extract_error(&json!({"code": "InvalidParameter", "message": "size invalid"})).unwrap(), + ( + Some("InvalidParameter".to_string()), + "size invalid".to_string() + ) + ); + assert_eq!( + extract_error( + &json!({"output": {"code": "DataInspectionFailed", "message": "blocked"}}) + ) + .unwrap(), + ( + Some("DataInspectionFailed".to_string()), + "blocked".to_string() + ) + ); + assert!(extract_error(&json!({"data": []})).is_none()); + } + + #[test] + fn content_filter_is_detected_from_the_provider_message() { + assert!(looks_like_content_filter( + "Your request was rejected by the safety system" + )); + assert!(looks_like_content_filter("DataInspectionFailed")); + assert!(!looks_like_content_filter("invalid size parameter")); + } + + #[test] + fn status_classification_maps_actionable_kinds() { + assert_eq!( + classify_status(reqwest::StatusCode::UNAUTHORIZED), + ImageGenerationErrorKind::Auth + ); + assert_eq!( + classify_status(reqwest::StatusCode::TOO_MANY_REQUESTS), + ImageGenerationErrorKind::RateLimited + ); + assert_eq!( + classify_status(reqwest::StatusCode::BAD_REQUEST), + ImageGenerationErrorKind::InvalidRequest + ); + assert_eq!( + classify_status(reqwest::StatusCode::INTERNAL_SERVER_ERROR), + ImageGenerationErrorKind::Provider + ); + } + + #[test] + fn dashscope_failed_task_status_is_reported() { + let value = json!({"output": {"task_status": "FAILED"}}); + assert_eq!( + dialect_reported_failure(ImageDialect::DashScopeNative, &value), + Some("FAILED".to_string()) + ); + assert_eq!( + dialect_reported_failure(ImageDialect::OpenAiImages, &value), + None + ); + } + + #[test] + fn mime_sniffing_recognizes_common_formats() { + assert_eq!(sniff_mime(&[0x89, b'P', b'N', b'G', 0]), Some("image/png")); + assert_eq!(sniff_mime(&[0xFF, 0xD8, 0xFF, 0]), Some("image/jpeg")); + assert_eq!(sniff_mime(b"RIFF0000WEBP"), Some("image/webp")); + assert_eq!(sniff_mime(b"not an image"), None); + } + + #[tokio::test] + async fn subscription_catalog_sends_bearer_and_account_headers() { + use axum::{ + http::{HeaderMap, StatusCode}, + routing::get, + Json, Router, + }; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let app = Router::new().route("/models", get(|headers: HeaderMap| async move { + if headers.get("authorization").and_then(|value| value.to_str().ok()) != Some("Bearer sk-test") + || headers.get("chatgpt-account-id").and_then(|value| value.to_str().ok()) != Some("test-account") + { + return (StatusCode::UNAUTHORIZED, Json(json!({"error": {"message": "missing subscription authentication"}}))); + } + (StatusCode::OK, Json(json!({"models": [{"slug": "account-model", "priority": 0}]}))) + })); + axum::serve(listener, app).await.unwrap(); + }); + let headers = + HashMap::from([("ChatGPT-Account-ID".to_string(), "test-account".to_string())]); + let mut target = target(ImageDialect::ChatgptCodex, &base_url); + target.custom_headers = Some(&headers); + openbitfun_services_core::tls_provider::ensure_ring_crypto_provider(); + let client = Client::builder().no_proxy().build().unwrap(); + let result = discover_chatgpt_codex_model(&client, &target).await; + server.abort(); + assert_eq!(result.unwrap(), "account-model"); + } + + #[cfg(feature = "subscription-auth")] + #[tokio::test] + #[ignore = "requires a real Codex login; performs one billed image generation"] + async fn probe_codex_subscription_image_generation() { + let options = crate::subscription_auth::SubscriptionHttpOptions::default(); + let credential = crate::subscription_auth::resolve_with_options( + crate::subscription_auth::SubscriptionProvider::Codex, + &options, + ) + .await + .expect("codex credential"); + let model = std::env::var("PROBE_CODEX_MODEL").unwrap_or_default(); + let target = ImageGenerationTarget { + dialect: ImageDialect::ChatgptCodex, + base_url: credential.base_url.as_deref().unwrap_or_default(), + request_url: credential.request_url.as_deref(), + model: &model, + api_key: &credential.api_key, + custom_headers: Some(&credential.extra_headers), + custom_headers_mode: None, + skip_ssl_verify: false, + proxy: None, + }; + let response = generate( + &target, + ImageGenerationRequest { + prompt: "A simple red circle on a white background".to_string(), + size: Some("1024x1024".to_string()), + n: Some(1), + ..ImageGenerationRequest::default() + }, + ) + .await + .expect("subscription image generation"); + assert!(!response.images.is_empty()); + for image in response.images { + assert!(sniff_mime(&image.bytes).is_some()); + println!( + "Generated image: mime={} bytes={}", + image.mime, + image.bytes.len() + ); + } + } + + #[tokio::test] + async fn generate_rejects_an_empty_prompt_before_any_request() { + let error = generate( + &target(ImageDialect::OpenAiImages, "https://api.openai.com/v1"), + ImageGenerationRequest::default(), + ) + .await + .unwrap_err(); + + assert_eq!(error.kind, ImageGenerationErrorKind::Config); + } +} diff --git a/src/crates/assembly/core/AGENTS.md b/src/crates/assembly/core/AGENTS.md index 8f073004db..4b8cfc7eac 100644 --- a/src/crates/assembly/core/AGENTS.md +++ b/src/crates/assembly/core/AGENTS.md @@ -210,6 +210,15 @@ AI client construction and subscription credential compatibility: cargo test -p openbitfun-core --no-default-features --features ai-adapter-runtime,subscription-auth --lib infrastructure::ai::client_factory::tests ``` +For image-service selection and compatibility with saved subscription configurations: + +```bash +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-image-analysis,subscription-auth --lib agentic::image_generation::tests +``` + +These fixtures do not call a provider or exercise live remote hosts. Image wire +protocol and opt-in live probes belong to the AI adapters guide. + This guide owns Core verification. Select one command pattern that matches the change; do not run every feature variant: diff --git a/src/crates/assembly/core/Cargo.toml b/src/crates/assembly/core/Cargo.toml index 573dd19d73..c4f1420803 100644 --- a/src/crates/assembly/core/Cargo.toml +++ b/src/crates/assembly/core/Cargo.toml @@ -377,6 +377,8 @@ tools-computer-use = [ ] tools-image-analysis = [ "openbitfun-tool-packs/image-analysis", + "ai-adapter-runtime", + "openbitfun-ai-adapters/image-generation", ] tools-miniapp = [ "dep:openbitfun-product-domains", diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/modes/claw.rs b/src/crates/assembly/core/src/agentic/agents/definitions/modes/claw.rs index f5109f7555..0fe198100e 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/modes/claw.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/modes/claw.rs @@ -25,6 +25,7 @@ impl ClawMode { "Read".to_string(), "view_image".to_string(), "analyze_image".to_string(), + "generate_image".to_string(), "Write".to_string(), "Edit".to_string(), "Delete".to_string(), diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/modes/cowork.rs b/src/crates/assembly/core/src/agentic/agents/definitions/modes/cowork.rs index 882e3d13af..ca284f1e18 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/modes/cowork.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/modes/cowork.rs @@ -36,6 +36,7 @@ impl CoworkMode { "Read".to_string(), "view_image".to_string(), "analyze_image".to_string(), + "generate_image".to_string(), "Grep".to_string(), "Glob".to_string(), "Write".to_string(), diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/modes/deep_research.rs b/src/crates/assembly/core/src/agentic/agents/definitions/modes/deep_research.rs index 19ac81a3eb..fbbdc7f555 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/modes/deep_research.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/modes/deep_research.rs @@ -29,6 +29,7 @@ impl DeepResearchMode { "Read".to_string(), "view_image".to_string(), "analyze_image".to_string(), + "generate_image".to_string(), "Grep".to_string(), "Glob".to_string(), "Write".to_string(), diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/subagents/general_purpose.rs b/src/crates/assembly/core/src/agentic/agents/definitions/subagents/general_purpose.rs index 4cafe747eb..f71910e2ba 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/subagents/general_purpose.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/subagents/general_purpose.rs @@ -18,6 +18,7 @@ impl GeneralPurposeAgent { "Read".to_string(), "view_image".to_string(), "analyze_image".to_string(), + "generate_image".to_string(), "Glob".to_string(), "Grep".to_string(), "Write".to_string(), diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/subagents/swarm.rs b/src/crates/assembly/core/src/agentic/agents/definitions/subagents/swarm.rs index 89a9923186..8b182a0d38 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/subagents/swarm.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/subagents/swarm.rs @@ -137,6 +137,7 @@ impl SwarmWorkerAgent { "Read", "view_image", "analyze_image", + "generate_image", "Glob", "Grep", "Write", diff --git a/src/crates/assembly/core/src/agentic/agents/mod.rs b/src/crates/assembly/core/src/agentic/agents/mod.rs index 057bcaa9ca..b1d6c50ed8 100644 --- a/src/crates/assembly/core/src/agentic/agents/mod.rs +++ b/src/crates/assembly/core/src/agentic/agents/mod.rs @@ -94,6 +94,7 @@ pub fn standard_harness_tools() -> Vec { "Read".to_string(), "view_image".to_string(), "analyze_image".to_string(), + "generate_image".to_string(), "Write".to_string(), "Edit".to_string(), "Delete".to_string(), diff --git a/src/crates/assembly/core/src/agentic/image_generation/mod.rs b/src/crates/assembly/core/src/agentic/image_generation/mod.rs new file mode 100644 index 0000000000..a85c2b90e2 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/image_generation/mod.rs @@ -0,0 +1,595 @@ +//! Resolution of the configured image generation service. +//! +//! The image service is selected in settings and is independent of the session's +//! text model: a session never decides which image provider is called. This +//! module is the single place that turns the persisted `media` section into a +//! callable adapter target. + +use crate::infrastructure::ai::client_factory::apply_subscription_auth; +use crate::service::config::get_global_config_service; +use crate::service::config::types::{ + AIConfig as ServiceAIConfig, AuthConfig, ImageServiceConfig, MediaConfig, SubscriptionProvider, +}; +use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; +use openbitfun_ai_adapters::client::image_generation::{ + generate, GeneratedImage, ImageGenerationError, ImageGenerationErrorKind, + ImageGenerationRequest, ImageGenerationTarget, +}; +use openbitfun_core_types::{AIConfig, ImageDialect, ProxyConfig}; +use std::collections::HashMap; + +/// Authentication material for one image call. +/// +/// An image service can authenticate with its own API key or with an imported +/// subscription login. Resolving the login produces a bearer token and the +/// provider's extra headers, so both sources reach the adapter the same way. +#[derive(Debug, Clone, Default)] +pub struct ImageServiceCredentials { + pub api_key: String, + pub custom_headers: Option>, + pub custom_headers_mode: Option, + /// Endpoint supplied by a subscription login. Only used when the service + /// itself has no base URL, because an image service owns its own dialect + /// endpoint rather than inheriting the chat endpoint. + pub base_url: Option, +} + +impl ImageServiceCredentials { + fn from_service(service: &ImageServiceConfig) -> Self { + Self { + api_key: service.api_key.clone(), + custom_headers: service.custom_headers.clone(), + custom_headers_mode: service.custom_headers_mode.clone(), + base_url: None, + } + } +} + +/// One resolved service plus the transport settings it inherits. +#[derive(Debug, Clone)] +pub struct ResolvedImageService { + pub service: ImageServiceConfig, + pub credentials: ImageServiceCredentials, + pub proxy: Option, +} + +fn no_service_error() -> OpenBitFunError { + OpenBitFunError::tool( + "No image generation service is configured. Add one in Settings > Models > Image generation." + .to_string(), + ) +} + +/// Resolve the credential material for one service. +/// +/// An API-key service needs no resolution. A subscription service reuses the +/// same login store the chat models use; when that login cannot be read the call +/// fails with the resolver's reason instead of sending an unauthenticated +/// request. +pub async fn resolve_credentials( + service: &ImageServiceConfig, +) -> OpenBitFunResult { + if matches!(service.auth, AuthConfig::SubscriptionPool { .. }) { + return Err(OpenBitFunError::tool( + "Image generation does not support subscription account pools; select a single account", + )); + } + if !matches!(service.auth, AuthConfig::Subscription { .. }) { + return Ok(ImageServiceCredentials::from_service(service)); + } + + let mut ai_config = transient_ai_config(service); + apply_subscription_auth(&service.auth, &mut ai_config) + .await + .map_err(|err| { + OpenBitFunError::tool(format!( + "Image service '{}' uses a subscription login that could not be resolved: {err}", + service.name.trim() + )) + })?; + + let subscription_base_url = ai_config.base_url.trim().to_string(); + // The subscription dialect is served from the provider's own backend, not + // from the address a user would type for an API key. Keeping a typed-in + // platform URL would send a subscription token to the wrong host, which the + // provider answers with a missing-scope error rather than a routing error. + let use_subscription_base_url = match service.dialect { + ImageDialect::ChatgptCodex => !subscription_base_url.is_empty(), + _ => service.base_url.trim().is_empty() && !subscription_base_url.is_empty(), + }; + Ok(ImageServiceCredentials { + api_key: ai_config.api_key, + custom_headers: ai_config.custom_headers, + custom_headers_mode: ai_config.custom_headers_mode, + base_url: use_subscription_base_url.then_some(subscription_base_url), + }) +} + +/// Minimal runtime config used to resolve a subscription login. +/// +/// Only the fields the resolver reads are populated; image generation never +/// sends a chat request with this config. +fn transient_ai_config(service: &ImageServiceConfig) -> AIConfig { + AIConfig { + name: service.name.clone(), + base_url: service.base_url.clone(), + request_url: service.request_url.clone().unwrap_or_default(), + api_key: service.api_key.clone(), + model: service.model_name.clone(), + format: String::new(), + context_window: 0, + max_tokens: None, + temperature: None, + top_p: None, + inline_think_in_text: false, + custom_headers: service.custom_headers.clone(), + custom_headers_mode: service.custom_headers_mode.clone(), + skip_ssl_verify: service.skip_ssl_verify, + custom_request_body: None, + custom_request_body_mode: None, + } +} + +/// Older credential-reuse UI saved Codex logins with the public image dialect. +/// Recover the callable view without rewriting or dropping the stored service. +fn service_with_auth_route(service: &ImageServiceConfig) -> ImageServiceConfig { + let mut service = service.clone(); + if service.dialect == ImageDialect::OpenAiImages + && matches!( + service.auth, + AuthConfig::Subscription { + provider: SubscriptionProvider::Codex, + .. + } + ) + { + service.dialect = ImageDialect::ChatgptCodex; + service.base_url.clear(); + service.request_url = None; + // An image-model ID cannot drive a Responses tool call. Resolve a chat + // model from this subscription's live catalog instead. + service.model_name.clear(); + service.params.n = Some(1); + } + service +} + +/// Pure resolution so the selection rules are testable without global config. +pub fn resolve_from_media( + media: &MediaConfig, + proxy: Option, + requested: Option<&str>, +) -> OpenBitFunResult { + let service = match requested.map(str::trim).filter(|id| !id.is_empty()) { + Some(id) => media.enabled_image_service(id).ok_or_else(|| { + OpenBitFunError::tool(format!( + "Image generation service '{id}' is not configured or is disabled." + )) + })?, + None => { + if media.default_image_service.is_none() { + let enabled = media + .image_services + .iter() + .filter(|service| service.enabled) + .count(); + if enabled > 1 { + return Err(OpenBitFunError::tool( + "Multiple image generation services are enabled but no default is selected. Set one in Settings > Models > Image generation." + .to_string(), + )); + } + } + media + .effective_image_service() + .ok_or_else(no_service_error)? + } + }; + + let service = service_with_auth_route(service); + let credentials = ImageServiceCredentials::from_service(&service); + Ok(ResolvedImageService { + service, + credentials, + proxy, + }) +} + +/// Resolve the service the `generate_image` tool should call. +pub async fn resolve_image_service( + requested: Option<&str>, +) -> OpenBitFunResult { + let config_service = get_global_config_service().await?; + let media: MediaConfig = config_service + .get_config(Some("media")) + .await + .map_err(|err| { + OpenBitFunError::service(format!("Failed to read image generation config: {err}")) + })?; + + // Media services inherit the global proxy; a per-service proxy is not a + // separate user-facing setting. + let mut resolved = resolve_from_media(&media, configured_proxy().await, requested)?; + resolved.credentials = resolve_credentials(&resolved.service).await?; + Ok(resolved) +} + +/// Whether any usable image service exists. Used to keep the tool out of the +/// model's tool list when generation would only be able to fail. +pub async fn has_enabled_image_service() -> bool { + let Ok(config_service) = get_global_config_service().await else { + return false; + }; + config_service + .get_config::(Some("media")) + .await + .map(|media| media.image_services.iter().any(|service| service.enabled)) + .unwrap_or(false) +} + +/// Convert a resolved service into an adapter target for one call. +pub fn adapter_target(service: &ResolvedImageService) -> ImageGenerationTarget<'_> { + target_for_credentials( + &service.service, + &service.credentials, + service.proxy.clone(), + ) +} + +/// Adapter target for one explicit service config and its credential material. +/// +/// The settings test action uses this so it exercises exactly the same +/// transport path as the `generate_image` tool, including the resolved +/// subscription login, proxy, custom headers and TLS policy. +pub fn target_for_credentials<'a>( + service: &'a ImageServiceConfig, + credentials: &'a ImageServiceCredentials, + proxy: Option, +) -> ImageGenerationTarget<'a> { + // A service owns its dialect endpoint. A subscription-supplied endpoint is + // only a fallback for a draft that leaves the address empty, except on the + // subscription dialect, whose address belongs to the login rather than to + // anything a user types here. + let subscription_endpoint_wins = credentials.base_url.is_some() + && (service.dialect == ImageDialect::ChatgptCodex || service.base_url.trim().is_empty()); + let base_url = if subscription_endpoint_wins { + credentials.base_url.as_deref().unwrap_or_default() + } else { + service.base_url.as_str() + }; + + ImageGenerationTarget { + dialect: service.dialect, + base_url, + request_url: service.request_url.as_deref(), + model: &service.model_name, + api_key: &credentials.api_key, + custom_headers: credentials.custom_headers.as_ref(), + custom_headers_mode: credentials.custom_headers_mode.as_deref(), + skip_ssl_verify: service.skip_ssl_verify, + proxy, + } +} + +/// Request built from a service's persisted defaults and an explicit prompt. +/// +/// A settings test always asks for a single image so a check cannot quietly bill +/// for the configured `n`. +pub fn request_with_defaults(service: &ImageServiceConfig, prompt: &str) -> ImageGenerationRequest { + let params = &service.params; + ImageGenerationRequest { + prompt: prompt.to_string(), + negative_prompt: params.negative_prompt.clone(), + size: params.size.clone(), + n: Some(1), + quality: params.quality.clone(), + background: params.background.clone(), + output_format: params.output_format.clone(), + watermark: params.watermark, + prompt_extend: params.prompt_extend, + seed: params.seed, + } +} + +/// Outcome of one settings-page connectivity check. +#[derive(Debug, Clone, serde::Serialize)] +pub struct ImageServiceTestOutcome { + pub success: bool, + pub elapsed_ms: u64, + /// [`ImageGenerationErrorKind::wire_name`] when the call failed. + pub error_kind: Option, + pub message: Option, + pub mime: Option, + pub image_base64: Option, +} + +impl ImageServiceTestOutcome { + fn failure(elapsed_ms: u64, error: &ImageGenerationError) -> Self { + Self { + success: false, + elapsed_ms, + error_kind: Some(error.kind.wire_name().to_string()), + message: Some(error.message.clone()), + mime: None, + image_base64: None, + } + } +} + +/// Generate one real image with an unsaved service draft. +/// +/// This bills the user's provider, so the caller must present it as a real +/// generation rather than a free probe. +pub async fn test_service(service: &ImageServiceConfig, prompt: &str) -> ImageServiceTestOutcome { + let started = std::time::Instant::now(); + let service = service_with_auth_route(service); + let credentials = match resolve_credentials(&service).await { + Ok(credentials) => credentials, + Err(error) => { + return ImageServiceTestOutcome { + success: false, + elapsed_ms: started.elapsed().as_millis() as u64, + error_kind: Some(ImageGenerationErrorKind::Auth.wire_name().to_string()), + message: Some(error.to_string()), + mime: None, + image_base64: None, + } + } + }; + + let proxy = configured_proxy().await; + let target = target_for_credentials(&service, &credentials, proxy); + let request = request_with_defaults(&service, prompt); + + match generate(&target, request).await { + Ok(response) => { + let elapsed_ms = started.elapsed().as_millis() as u64; + match response.images.into_iter().next() { + Some(image) => success_outcome(elapsed_ms, image), + None => ImageServiceTestOutcome::failure( + elapsed_ms, + &ImageGenerationError { + kind: ImageGenerationErrorKind::EmptyResult, + message: "The provider returned no image".to_string(), + }, + ), + } + } + Err(error) => { + ImageServiceTestOutcome::failure(started.elapsed().as_millis() as u64, &error) + } + } +} + +fn success_outcome(elapsed_ms: u64, image: GeneratedImage) -> ImageServiceTestOutcome { + let base64 = image.base64(); + ImageServiceTestOutcome { + success: true, + elapsed_ms, + error_kind: None, + message: None, + mime: Some(image.mime), + image_base64: Some(base64), + } +} + +/// Global HTTP proxy inherited by media services, when one is configured. +async fn configured_proxy() -> Option { + let config_service = get_global_config_service().await.ok()?; + let ai = config_service + .get_config::(Some("ai")) + .await + .ok()?; + ai.proxy.enabled.then_some(ai.proxy) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service::config::types::{ImageDialect, ImageGenerationParams}; + + fn service(id: &str, enabled: bool) -> ImageServiceConfig { + ImageServiceConfig { + id: id.to_string(), + name: id.to_string(), + dialect: ImageDialect::OpenAiImages, + base_url: "https://api.openai.com/v1".to_string(), + model_name: "gpt-image-1".to_string(), + api_key: "sk-test".to_string(), + enabled, + params: ImageGenerationParams::default(), + ..ImageServiceConfig::default() + } + } + + fn media(services: Vec, default: Option<&str>) -> MediaConfig { + MediaConfig { + image_services: services, + default_image_service: default.map(str::to_string), + } + } + + #[test] + fn explicit_request_wins_over_the_default() { + let media = media( + vec![service("svc-a", true), service("svc-b", true)], + Some("svc-a"), + ); + + let resolved = resolve_from_media(&media, None, Some("svc-b")).unwrap(); + + assert_eq!(resolved.service.id, "svc-b"); + } + + #[test] + fn legacy_reused_codex_login_uses_its_subscription_route_without_mutating_storage() { + let mut saved = service("legacy", true); + saved.auth = AuthConfig::Subscription { + provider: SubscriptionProvider::Codex, + plan: None, + account_id: None, + }; + saved.request_url = Some("https://api.openai.com/v1/images/generations".to_string()); + let media: MediaConfig = serde_json::from_value(serde_json::json!({ + "image_services": [saved], "default_image_service": "legacy" + })) + .unwrap(); + let snapshot = serde_json::to_value(&media).unwrap(); + let resolved = resolve_from_media(&media, None, None).unwrap(); + assert_eq!(resolved.service.dialect, ImageDialect::ChatgptCodex); + assert!(resolved.service.model_name.is_empty()); + assert!(resolved.service.base_url.is_empty()); + assert_eq!(resolved.service.request_url, None); + assert_eq!(serde_json::to_value(&media).unwrap(), snapshot); + } + + #[tokio::test] + async fn account_pools_fail_explicitly_before_using_inline_api_credentials() { + let mut saved = service("pool", true); + saved.auth = AuthConfig::SubscriptionPool { + provider: SubscriptionProvider::Codex, + accounts: vec!["account-a".to_string()], + revision: 1, + }; + let error = resolve_credentials(&saved).await.unwrap_err(); + assert!(error + .to_string() + .contains("does not support subscription account pools")); + } + + #[test] + fn explicit_subscription_models_and_api_key_gateways_keep_their_configuration() { + let mut subscription = service("codex", true); + subscription.dialect = ImageDialect::ChatgptCodex; + subscription.auth = AuthConfig::Subscription { + provider: SubscriptionProvider::Codex, + plan: None, + account_id: None, + }; + subscription.model_name = "account-model".to_string(); + assert_eq!(service_with_auth_route(&subscription), subscription); + let mut api = service("gateway", true); + api.request_url = Some("https://images.example/generate".to_string()); + assert_eq!(service_with_auth_route(&api), api); + } + + #[test] + fn explicit_request_must_be_enabled() { + let media = media(vec![service("svc-a", false)], None); + + let error = resolve_from_media(&media, None, Some("svc-a")).unwrap_err(); + + assert!(error.to_string().contains("not configured or is disabled")); + } + + #[test] + fn default_service_is_used_when_no_request_names_one() { + let media = media( + vec![service("svc-a", true), service("svc-b", true)], + Some("svc-b"), + ); + + assert_eq!( + resolve_from_media(&media, None, None).unwrap().service.id, + "svc-b" + ); + } + + #[test] + fn a_single_enabled_service_needs_no_default() { + let media = media(vec![service("svc-a", true), service("svc-b", false)], None); + + assert_eq!( + resolve_from_media(&media, None, None).unwrap().service.id, + "svc-a" + ); + } + + #[test] + fn ambiguous_services_without_a_default_are_reported() { + let media = media(vec![service("svc-a", true), service("svc-b", true)], None); + + let error = resolve_from_media(&media, None, None).unwrap_err(); + + assert!(error.to_string().contains("no default is selected")); + } + + #[test] + fn no_services_reports_the_settings_entry_point() { + let error = resolve_from_media(&media(Vec::new(), None), None, None).unwrap_err(); + + assert!(error + .to_string() + .contains("Settings > Models > Image generation")); + } + + #[test] + fn an_api_key_service_resolves_its_own_credentials() { + let mut config = service("svc-a", true); + config.custom_headers = Some(HashMap::from([( + "x-tenant".to_string(), + "acme".to_string(), + )])); + + let credentials = ImageServiceCredentials::from_service(&config); + + assert_eq!(credentials.api_key, "sk-test"); + assert_eq!(credentials.base_url, None); + assert_eq!( + credentials + .custom_headers + .as_ref() + .and_then(|headers| headers.get("x-tenant")) + .map(String::as_str), + Some("acme") + ); + } + + #[test] + fn subscription_endpoint_only_fills_a_missing_service_base_url() { + let mut config = service("svc-a", true); + config.base_url = String::new(); + let credentials = ImageServiceCredentials { + api_key: "oauth-token".to_string(), + base_url: Some("https://chatgpt.example/backend".to_string()), + ..ImageServiceCredentials::default() + }; + + let target = target_for_credentials(&config, &credentials, None); + + assert_eq!(target.base_url, "https://chatgpt.example/backend"); + assert_eq!(target.api_key, "oauth-token"); + } + + #[test] + fn a_service_base_url_survives_a_subscription_login() { + let config = service("svc-a", true); + let credentials = ImageServiceCredentials { + api_key: "oauth-token".to_string(), + base_url: Some("https://chatgpt.example/backend".to_string()), + ..ImageServiceCredentials::default() + }; + + let target = target_for_credentials(&config, &credentials, None); + + assert_eq!(target.base_url, "https://api.openai.com/v1"); + } + + #[test] + fn adapter_target_carries_the_service_fields() { + let mut config = service("svc-a", true); + config.custom_headers_mode = Some("replace".to_string()); + let resolved = ResolvedImageService { + credentials: ImageServiceCredentials::from_service(&config), + service: config, + proxy: None, + }; + + let target = adapter_target(&resolved); + + assert_eq!(target.model, "gpt-image-1"); + assert_eq!(target.custom_headers_mode, Some("replace")); + assert_eq!(target.dialect, ImageDialect::OpenAiImages); + assert_eq!(target.api_key, "sk-test"); + } +} diff --git a/src/crates/assembly/core/src/agentic/mod.rs b/src/crates/assembly/core/src/agentic/mod.rs index b0ee8460dc..f2aff80509 100644 --- a/src/crates/assembly/core/src/agentic/mod.rs +++ b/src/crates/assembly/core/src/agentic/mod.rs @@ -33,6 +33,8 @@ pub mod round_preempt; // Image analysis module pub mod image_analysis; +#[cfg(feature = "tools-image-analysis")] +pub mod image_generation; pub(crate) mod keyed_lock; pub mod memories; pub(crate) mod permission_policy; diff --git a/src/crates/assembly/core/src/agentic/tools/agent-tool-exposure.md b/src/crates/assembly/core/src/agentic/tools/agent-tool-exposure.md index c71ef28a8e..b6f2d4f5a3 100644 --- a/src/crates/assembly/core/src/agentic/tools/agent-tool-exposure.md +++ b/src/crates/assembly/core/src/agentic/tools/agent-tool-exposure.md @@ -29,6 +29,7 @@ Notes: | `Read` | Direct | None | - | | `view_image` | Direct | None | - | | `analyze_image` | Direct | None | - | +| `generate_image` | Direct | None | - | | `Glob` | Direct | None | - | | `Grep` | Direct | None | - | | `Write` | Direct | None | - | diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs new file mode 100644 index 0000000000..a819564881 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs @@ -0,0 +1,488 @@ +use crate::agentic::image_generation::{ + adapter_target, has_enabled_image_service, resolve_image_service, ResolvedImageService, +}; +use crate::agentic::tools::framework::{ + Tool, ToolRenderOptions, ToolResult, ToolUseContext, ValidationResult, +}; +use crate::service::config::types::{ImageGenerationParams, ImageServiceConfig}; +use crate::util::errors::{OpenBitFunError, OpenBitFunResult}; +use async_trait::async_trait; +use openbitfun_ai_adapters::client::image_generation::{ + generate, GeneratedImage, ImageGenerationRequest, +}; +use openbitfun_core_types::MAX_IMAGES_PER_GENERATION_REQUEST; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use tokio::fs; + +/// Workspace-relative directory that receives generated images. +const OUTPUT_DIR_COMPONENTS: [&str; 2] = [".openbitfun", "generated-images"]; + +pub struct GenerateImageTool; + +impl Default for GenerateImageTool { + fn default() -> Self { + Self::new() + } +} + +/// One image already written to disk. +struct PersistedImage { + path: String, + mime: String, + bytes: usize, +} + +impl GenerateImageTool { + pub fn new() -> Self { + Self + } + + fn prompt_from_input(input: &Value) -> OpenBitFunResult<&str> { + input + .get("prompt") + .and_then(Value::as_str) + .map(str::trim) + .filter(|prompt| !prompt.is_empty()) + .ok_or_else(|| OpenBitFunError::tool("prompt is required".to_string())) + } + + fn requested_count(input: &Value) -> OpenBitFunResult> { + match input.get("n") { + None | Some(Value::Null) => Ok(None), + Some(value) => { + let Some(count) = value.as_u64() else { + return Err(OpenBitFunError::tool( + "n must be a positive integer".to_string(), + )); + }; + if count == 0 || count > u64::from(MAX_IMAGES_PER_GENERATION_REQUEST) { + return Err(OpenBitFunError::tool(format!( + "n must be between 1 and {MAX_IMAGES_PER_GENERATION_REQUEST}" + ))); + } + Ok(Some(count as u32)) + } + } + } + + /// Service parameters are defaults; a per-call argument overrides them. + fn build_request( + service: &ImageServiceConfig, + input: &Value, + ) -> OpenBitFunResult { + let params: &ImageGenerationParams = &service.params; + let size = input + .get("size") + .and_then(Value::as_str) + .or(params.size.as_deref()) + .map(str::to_string); + let negative_prompt = input + .get("negative_prompt") + .and_then(Value::as_str) + .or(params.negative_prompt.as_deref()) + .map(str::to_string); + + Ok(ImageGenerationRequest { + prompt: Self::prompt_from_input(input)?.to_string(), + negative_prompt, + size, + n: Self::requested_count(input)?.or(params.n), + quality: params.quality.clone(), + background: params.background.clone(), + output_format: params.output_format.clone(), + watermark: params.watermark, + prompt_extend: params.prompt_extend, + seed: params.seed, + }) + } + + fn extension_for(mime: &str) -> &'static str { + match mime { + "image/jpeg" => "jpg", + "image/webp" => "webp", + "image/gif" => "gif", + _ => "png", + } + } + + fn file_name(mime: &str) -> String { + format!("{}.{}", uuid::Uuid::new_v4(), Self::extension_for(mime)) + } + + /// Write one image into the workspace so the session keeps a real file and + /// the chat card can preview it. Remote workspaces write through the + /// workspace port, which owns its own path syntax. + async fn persist_one( + context: &ToolUseContext, + image: &GeneratedImage, + ) -> OpenBitFunResult { + let file_name = Self::file_name(&image.mime); + + if context.is_remote() { + let fs = context.ws_fs().ok_or_else(|| { + OpenBitFunError::tool( + "generate_image cannot write to the remote workspace because workspace filesystem services are unavailable" + .to_string(), + ) + })?; + let root = context + .workspace_root() + .map(|root| root.to_string_lossy().into_owned()) + .ok_or_else(|| { + OpenBitFunError::tool( + "generate_image requires a workspace to store the generated image" + .to_string(), + ) + })?; + let directory = fs.join_path(&root, &OUTPUT_DIR_COMPONENTS); + fs.create_dir_all(&directory).await.map_err(|err| { + OpenBitFunError::tool(format!( + "unable to create the remote image directory {directory}: {err}" + )) + })?; + let path = fs.join_path(&directory, &[file_name.as_str()]); + fs.write_file(&path, &image.bytes).await.map_err(|err| { + OpenBitFunError::tool(format!("unable to write the generated image {path}: {err}")) + })?; + return Ok(PersistedImage { + path, + mime: image.mime.clone(), + bytes: image.bytes.len(), + }); + } + + let directory = match context.workspace_root() { + Some(root) => Self::local_directory(root), + None => std::env::temp_dir(), + }; + fs::create_dir_all(&directory).await.map_err(|err| { + OpenBitFunError::tool(format!( + "unable to create the image directory {}: {err}", + directory.display() + )) + })?; + let path = directory.join(&file_name); + fs::write(&path, &image.bytes).await.map_err(|err| { + OpenBitFunError::tool(format!( + "unable to write the generated image {}: {err}", + path.display() + )) + })?; + + Ok(PersistedImage { + path: path.to_string_lossy().into_owned(), + mime: image.mime.clone(), + bytes: image.bytes.len(), + }) + } + + fn local_directory(root: &Path) -> PathBuf { + let mut directory = root.to_path_buf(); + for component in OUTPUT_DIR_COMPONENTS { + directory = directory.join(component); + } + directory + } + + fn result_summary(resolved: &ResolvedImageService, persisted: &[PersistedImage]) -> String { + let noun = if persisted.len() == 1 { + "image" + } else { + "images" + }; + format!( + "Generated {} {noun} with {} ({})", + persisted.len(), + resolved.service.name, + persisted + .iter() + .map(|image| image.path.as_str()) + .collect::>() + .join(", ") + ) + } +} + +#[async_trait] +impl Tool for GenerateImageTool { + fn name(&self) -> &str { + "generate_image" + } + + async fn description(&self) -> OpenBitFunResult { + Ok( + "Generate images from a text prompt using the image service selected in Settings > Image Generation. Use this whenever the user asks for a picture, illustration, icon, poster or any other generated image. Returns the saved file paths." + .to_string(), + ) + } + + fn short_description(&self) -> String { + "Generate an image from a text prompt.".to_string() + } + + fn input_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "prompt": { + "type": "string", + "description": "What the image should show. Describe subject, style, composition and lighting." + }, + "size": { + "type": "string", + "description": "Optional size override in the configured service's own syntax, such as 1024x1024. Omit to use the service default." + }, + "n": { + "type": "integer", + "minimum": 1, + "maximum": MAX_IMAGES_PER_GENERATION_REQUEST, + "description": "Number of images to generate. Omit to use the service default." + }, + "negative_prompt": { + "type": "string", + "description": "Optional content to avoid. Only honored by services that support it." + }, + "service_id": { + "type": "string", + "description": "Optional image service id. Omit to use the default service." + } + }, + "required": ["prompt"], + "additionalProperties": false + }) + } + + async fn is_available_in_context(&self, _context: Option<&ToolUseContext>) -> bool { + has_enabled_image_service().await + } + + fn is_readonly(&self) -> bool { + false + } + + async fn validate_input( + &self, + input: &Value, + _context: Option<&ToolUseContext>, + ) -> ValidationResult { + if let Err(err) = Self::prompt_from_input(input) { + return ValidationResult { + result: false, + message: Some(err.to_string()), + error_code: Some(400), + meta: None, + }; + } + if let Err(err) = Self::requested_count(input) { + return ValidationResult { + result: false, + message: Some(err.to_string()), + error_code: Some(400), + meta: None, + }; + } + ValidationResult::default() + } + + fn render_tool_use_message(&self, input: &Value, _options: &ToolRenderOptions) -> String { + match Self::prompt_from_input(input) { + Ok(prompt) => { + let preview: String = prompt.chars().take(80).collect(); + if preview.len() < prompt.len() { + format!("Generating image: {preview}...") + } else { + format!("Generating image: {preview}") + } + } + Err(_) => "Generating image".to_string(), + } + } + + fn render_tool_result_message(&self, output: &Value) -> String { + let count = output + .get("count") + .and_then(Value::as_u64) + .unwrap_or_default(); + match count { + 0 => "No image generated".to_string(), + 1 => "Generated 1 image".to_string(), + other => format!("Generated {other} images"), + } + } + + async fn call_impl( + &self, + input: &Value, + context: &ToolUseContext, + ) -> OpenBitFunResult> { + let service_id = input + .get("service_id") + .and_then(Value::as_str) + .map(str::trim) + .filter(|id| !id.is_empty()); + let resolved = resolve_image_service(service_id).await?; + let request = Self::build_request(&resolved.service, input)?; + + let target = adapter_target(&resolved); + let response = generate(&target, request) + .await + .map_err(|err| OpenBitFunError::tool(format!("Image generation failed: {err}")))?; + + let mut persisted = Vec::with_capacity(response.images.len()); + for image in &response.images { + persisted.push(Self::persist_one(context, image).await?); + } + if persisted.is_empty() { + return Err(OpenBitFunError::tool( + "Image generation returned no images".to_string(), + )); + } + + let summary = Self::result_summary(&resolved, &persisted); + let paths: Vec<&str> = persisted.iter().map(|image| image.path.as_str()).collect(); + let data = json!({ + "path": persisted[0].path, + "paths": paths, + "count": persisted.len(), + "service_id": resolved.service.id, + "service": resolved.service.name, + "dialect": format!("{:?}", resolved.service.dialect), + "model": resolved.service.model_name, + "mime_type": persisted[0].mime, + "bytes": persisted[0].bytes, + "summary": summary, + }); + + Ok(vec![ToolResult::ok(data, Some(summary))]) + } +} + +#[cfg(test)] +mod tests { + use super::GenerateImageTool; + use crate::agentic::tools::framework::Tool; + use crate::service::config::types::{ImageDialect, ImageGenerationParams, ImageServiceConfig}; + use serde_json::json; + + fn service(params: ImageGenerationParams) -> ImageServiceConfig { + ImageServiceConfig { + id: "svc-a".to_string(), + name: "Test service".to_string(), + dialect: ImageDialect::DashScopeNative, + base_url: "https://dashscope.aliyuncs.com/api/v1".to_string(), + model_name: "qwen-image-2.0".to_string(), + api_key: "sk-test".to_string(), + enabled: true, + params, + ..ImageServiceConfig::default() + } + } + + #[test] + fn service_parameters_are_the_defaults() { + let service = service(ImageGenerationParams { + size: Some("2048*2048".to_string()), + n: Some(2), + negative_prompt: Some("blur".to_string()), + watermark: Some(false), + ..ImageGenerationParams::default() + }); + + let request = + GenerateImageTool::build_request(&service, &json!({ "prompt": "a fox" })).unwrap(); + + assert_eq!(request.prompt, "a fox"); + assert_eq!(request.size.as_deref(), Some("2048*2048")); + assert_eq!(request.n, Some(2)); + assert_eq!(request.negative_prompt.as_deref(), Some("blur")); + assert_eq!(request.watermark, Some(false)); + } + + #[test] + fn call_arguments_override_service_parameters() { + let service = service(ImageGenerationParams { + size: Some("2048*2048".to_string()), + n: Some(2), + negative_prompt: Some("blur".to_string()), + ..ImageGenerationParams::default() + }); + + let request = GenerateImageTool::build_request( + &service, + &json!({ "prompt": "a fox", "size": "1024*1024", "n": 1, "negative_prompt": "noise" }), + ) + .unwrap(); + + assert_eq!(request.size.as_deref(), Some("1024*1024")); + assert_eq!(request.n, Some(1)); + assert_eq!(request.negative_prompt.as_deref(), Some("noise")); + } + + #[test] + fn a_missing_prompt_is_rejected() { + let error = GenerateImageTool::build_request( + &service(ImageGenerationParams::default()), + &json!({}), + ) + .unwrap_err(); + + assert!(error.to_string().contains("prompt is required")); + } + + #[test] + fn a_blank_prompt_is_rejected() { + let error = GenerateImageTool::build_request( + &service(ImageGenerationParams::default()), + &json!({ "prompt": " " }), + ) + .unwrap_err(); + + assert!(error.to_string().contains("prompt is required")); + } + + #[test] + fn an_out_of_range_count_is_rejected() { + for value in [json!(0), json!(11)] { + let error = GenerateImageTool::build_request( + &service(ImageGenerationParams::default()), + &json!({ "prompt": "a fox", "n": value }), + ) + .unwrap_err(); + assert!(error.to_string().contains("n must be between")); + } + } + + #[tokio::test] + async fn validate_input_reports_a_missing_prompt() { + let validation = GenerateImageTool::new() + .validate_input(&json!({}), None) + .await; + + assert!(!validation.result); + assert_eq!(validation.error_code, Some(400)); + } + + #[test] + fn file_extension_follows_the_mime_type() { + assert_eq!(GenerateImageTool::extension_for("image/png"), "png"); + assert_eq!(GenerateImageTool::extension_for("image/jpeg"), "jpg"); + assert_eq!(GenerateImageTool::extension_for("image/webp"), "webp"); + assert_eq!( + GenerateImageTool::extension_for("application/octet-stream"), + "png" + ); + } + + #[test] + fn generated_files_live_under_the_workspace_openbitfun_directory() { + let directory = GenerateImageTool::local_directory(std::path::Path::new("/repo")); + + assert_eq!( + directory, + std::path::Path::new("/repo") + .join(".openbitfun") + .join("generated-images") + ); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs index e98569eab8..3885dc7647 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs @@ -36,6 +36,8 @@ pub mod file_read_tool; pub mod file_write_tool; #[cfg(feature = "tools-creation")] pub mod frontend_workbench_tool; +#[cfg(feature = "tools-image-analysis")] +pub mod generate_image_tool; #[cfg(feature = "tools-miniapp")] pub mod generative_ui_tool; #[cfg(feature = "tools-git")] @@ -111,6 +113,8 @@ pub use file_read_tool::FileReadTool; pub use file_write_tool::FileWriteTool; #[cfg(feature = "tools-creation")] pub use frontend_workbench_tool::FrontendWorkbenchTool; +#[cfg(feature = "tools-image-analysis")] +pub use generate_image_tool::GenerateImageTool; #[cfg(feature = "tools-miniapp")] pub use generative_ui_tool::GenerativeUITool; #[cfg(feature = "tools-git")] diff --git a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs index 66c47fbffb..fa18983803 100644 --- a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs +++ b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs @@ -21,6 +21,7 @@ const PRODUCT_TOOL_REGISTRATION_ORDER: &[&str] = &[ "Read", "view_image", "analyze_image", + "generate_image", "Glob", "Grep", "Write", @@ -111,6 +112,8 @@ impl StaticToolProviderFactory for ProductConcreteToolFactory { "view_image" => Some(Arc::new(ViewImageTool::new())), #[cfg(feature = "tools-image-analysis")] "analyze_image" => Some(Arc::new(AnalyzeImageTool::new())), + #[cfg(feature = "tools-image-analysis")] + "generate_image" => Some(Arc::new(GenerateImageTool::new())), "Glob" => Some(Arc::new(GlobTool::new())), "Grep" => Some(Arc::new(GrepTool::new())), "Write" => Some(Arc::new(FileWriteTool::new())), diff --git a/src/crates/assembly/core/src/agentic/tools/registry.rs b/src/crates/assembly/core/src/agentic/tools/registry.rs index 4883bd68b6..38e3b93155 100644 --- a/src/crates/assembly/core/src/agentic/tools/registry.rs +++ b/src/crates/assembly/core/src/agentic/tools/registry.rs @@ -559,6 +559,7 @@ mod tests { "Read", "view_image", "analyze_image", + "generate_image", "Glob", "Grep", "Write", diff --git a/src/crates/assembly/core/src/service/config/providers.rs b/src/crates/assembly/core/src/service/config/providers.rs index b81c509eee..3b11389461 100644 --- a/src/crates/assembly/core/src/service/config/providers.rs +++ b/src/crates/assembly/core/src/service/config/providers.rs @@ -674,6 +674,90 @@ impl ConfigProvider for AppConfigProvider { } } +/// Media (image generation) service configuration provider. +pub struct MediaConfigProvider; + +#[async_trait] +impl ConfigProvider for MediaConfigProvider { + fn name(&self) -> &str { + "media" + } + + fn get_default_config(&self) -> serde_json::Value { + serialize_default_config("media", MediaConfig::default()) + } + + async fn validate_config(&self, config: &serde_json::Value) -> OpenBitFunResult> { + let media = match serde_json::from_value::(config.clone()) { + Ok(media) => media, + Err(err) => { + return Err(OpenBitFunError::validation(format!( + "Invalid media config format: {}", + err + ))) + } + }; + + let mut warnings = Vec::new(); + for (index, service) in media.image_services.iter().enumerate() { + if service.id.trim().is_empty() { + return Err(OpenBitFunError::validation(format!( + "media.image_services[{index}]: service id is required" + ))); + } + if !service.enabled { + continue; + } + // The subscription dialect discovers its model from the account's + // catalog, so an empty field there is a supported choice. + if service.model_name.trim().is_empty() && !service.dialect.requires_subscription() { + warnings.push(format!( + "media.image_services[{index}]: model name is empty" + )); + } + if service.base_url.trim().is_empty() + && service.request_url.is_none() + && !matches!(service.auth, AuthConfig::Subscription { .. }) + { + warnings.push(format!("media.image_services[{index}]: base url is empty")); + } + let max_images = service.dialect.max_images_per_request(); + if let Some(n) = service.params.n { + if n == 0 || n > max_images { + warnings.push(format!( + "media.image_services[{index}]: n must be between 1 and {max_images}" + )); + } + } + } + + if let Some(default_id) = media.default_image_service.as_deref() { + if media.enabled_image_service(default_id).is_none() { + warnings.push(format!( + "media.default_image_service: '{default_id}' is not an enabled image service" + )); + } + } + + Ok(warnings) + } + + async fn on_config_changed( + &self, + _old_config: &serde_json::Value, + new_config: &serde_json::Value, + ) -> OpenBitFunResult<()> { + if let Ok(media) = serde_json::from_value::(new_config.clone()) { + info!( + "Media config changed: image_services={}, default_image_service={:?}", + media.image_services.len(), + media.default_image_service + ); + } + Ok(()) + } +} + /// Configuration provider registry. pub struct ConfigProviderRegistry { providers: HashMap>, @@ -692,6 +776,7 @@ impl ConfigProviderRegistry { registry.register(Box::new(TerminalConfigProvider)); registry.register(Box::new(WorkspaceConfigProvider)); registry.register(Box::new(AppConfigProvider)); + registry.register(Box::new(MediaConfigProvider)); registry } @@ -725,7 +810,15 @@ impl ConfigProviderRegistry { let mut errors = Vec::new(); let mut warnings = Vec::new(); - for provider_name in ["ai", "appearance", "editor", "terminal", "workspace", "app"] { + for provider_name in [ + "ai", + "appearance", + "editor", + "terminal", + "workspace", + "app", + "media", + ] { let Some(provider) = self.get_provider(provider_name) else { continue; }; @@ -818,6 +911,7 @@ impl ConfigProviderRegistry { "terminal" => Ok(serde_json::to_value(&config.terminal)?), "workspace" => Ok(serde_json::to_value(&config.workspace)?), "ai" => Ok(serde_json::to_value(&config.ai)?), + "media" => Ok(serde_json::to_value(&config.media)?), _ => Err(OpenBitFunError::validation(format!( "Unknown config section: {}", section @@ -1094,4 +1188,119 @@ mod tests { .message .contains("budget_tokens value must be greater than 0")); } + + fn image_service(id: &str, enabled: bool) -> ImageServiceConfig { + ImageServiceConfig { + id: id.to_string(), + name: id.to_string(), + dialect: ImageDialect::OpenAiImages, + base_url: "https://api.openai.com/v1".to_string(), + model_name: "gpt-image-1".to_string(), + api_key: "sk-test".to_string(), + enabled, + ..ImageServiceConfig::default() + } + } + + #[tokio::test] + async fn media_validation_accepts_a_subscription_service_without_a_base_url() { + let mut service = image_service("svc-a", true); + service.base_url = String::new(); + service.api_key = String::new(); + service.auth = AuthConfig::Subscription { + provider: SubscriptionProvider::Codex, + plan: None, + account_id: None, + }; + let mut config = GlobalConfig::default(); + config.media.image_services.push(service); + config.media.default_image_service = Some("svc-a".to_string()); + + let validation = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .expect("registry validation result"); + + assert!(validation.valid, "{:?}", validation.errors); + assert!(validation.warnings.is_empty(), "{:?}", validation.warnings); + } + + #[tokio::test] + async fn media_validation_accepts_a_clean_section() { + let mut config = GlobalConfig::default(); + config + .media + .image_services + .push(image_service("svc-a", true)); + config.media.default_image_service = Some("svc-a".to_string()); + + let validation = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .expect("registry validation result"); + + assert!(validation.valid, "{:?}", validation.errors); + assert!(validation.warnings.is_empty(), "{:?}", validation.warnings); + } + + #[tokio::test] + async fn media_validation_rejects_a_service_without_an_id() { + let mut config = GlobalConfig::default(); + config.media.image_services.push(image_service("", true)); + + let validation = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .expect("registry validation result"); + + assert!(!validation.valid); + assert_eq!(validation.errors[0].path, "media"); + assert!(validation.errors[0] + .message + .contains("service id is required")); + } + + #[tokio::test] + async fn media_validation_warns_on_a_dangling_default_pointer() { + let mut config = GlobalConfig::default(); + config + .media + .image_services + .push(image_service("svc-a", false)); + config.media.default_image_service = Some("missing".to_string()); + + let validation = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .expect("registry validation result"); + + assert!(validation.valid, "{:?}", validation.errors); + assert!(validation + .warnings + .iter() + .any(|warning| warning.message.contains("default_image_service"))); + } + + #[tokio::test] + async fn media_validation_warns_on_an_out_of_range_image_count() { + let mut config = GlobalConfig::default(); + config.media.image_services.push(ImageServiceConfig { + params: ImageGenerationParams { + n: Some(MAX_IMAGES_PER_GENERATION_REQUEST + 1), + ..ImageGenerationParams::default() + }, + ..image_service("svc-a", true) + }); + + let validation = ConfigProviderRegistry::new() + .validate_config(&config) + .await + .expect("registry validation result"); + + assert!(validation.valid); + assert!(validation + .warnings + .iter() + .any(|warning| warning.message.contains("n must be between"))); + } } diff --git a/src/crates/contracts/config-contracts/src/types.rs b/src/crates/contracts/config-contracts/src/types.rs index 1caa260842..33b2e5e06b 100644 --- a/src/crates/contracts/config-contracts/src/types.rs +++ b/src/crates/contracts/config-contracts/src/types.rs @@ -3,7 +3,10 @@ //! Defines all configuration-related types shared between backend and frontend. use openbitfun_core_types::{product_identity, ToolPresentationMode, WorktreeSettings}; -pub use openbitfun_core_types::{ReasoningConfig, ReasoningPreset, ReasoningPresetAction}; +pub use openbitfun_core_types::{ + ImageDialect, ReasoningConfig, ReasoningPreset, ReasoningPresetAction, + MAX_IMAGES_PER_GENERATION_REQUEST, +}; use openbitfun_product_domains::tool_permissions::{PermissionRule, ToolPermissionConfig}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -58,6 +61,12 @@ pub struct GlobalConfig { pub terminal: TerminalConfig, pub workspace: WorkspaceConfig, pub ai: AIConfig, + /// Image and other media service configuration. + /// + /// Always serialized, even when empty: the settings surface reads this path + /// through the generic dot-path getter, which fails on an absent key. + #[serde(default)] + pub media: MediaConfig, /// User-level static tool permission policy and interaction preferences. #[serde(default)] pub tool_permissions: ToolPermissionConfig, @@ -879,6 +888,110 @@ pub struct DefaultModelsConfig { pub speech_recognition: Option, } +/// Per-service generation parameters. They act as `generate_image` tool defaults +/// and can be overridden per call. +/// +/// `size` stays a dialect-native expression on purpose: a new provider then only +/// needs a UI preset instead of a contract change. `OpenAiImages` uses `1024x1024` +/// while `DashScopeNative` uses `2048*2048`. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ImageGenerationParams { + /// Dialect-native size expression. + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + /// Number of images requested per call. + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub negative_prompt: Option, + /// OpenAI Images quality preset (`auto` / `low` / `medium` / `high`). + #[serde(skip_serializing_if = "Option::is_none")] + pub quality: Option, + /// OpenAI Images background (`auto` / `opaque` / `transparent`). + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + /// Output encoding (`png` / `jpeg` / `webp`). + #[serde(skip_serializing_if = "Option::is_none")] + pub output_format: Option, + /// DashScope watermark flag. + #[serde(skip_serializing_if = "Option::is_none")] + pub watermark: Option, + /// DashScope prompt rewrite flag. + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_extend: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, +} + +/// One independently configured image generation service. +/// +/// This deliberately does not reuse [`AIModelConfig`]: an image service has no +/// context window, no token limits and no chat capability tags, and its request +/// shape is owned by [`ImageDialect`] rather than by the model name. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct ImageServiceConfig { + /// Stable identifier referenced by [`MediaConfig::default_image_service`]. + pub id: String, + pub name: String, + pub dialect: ImageDialect, + /// Service root without the dialect's endpoint path. + pub base_url: String, + /// Full request URL override. When set it replaces `base_url` + dialect path. + #[serde(skip_serializing_if = "Option::is_none")] + pub request_url: Option, + pub model_name: String, + pub api_key: String, + #[serde(default)] + pub auth: AuthConfig, + pub enabled: bool, + pub params: ImageGenerationParams, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_headers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_headers_mode: Option, + pub skip_ssl_verify: bool, +} + +/// Media service configuration. +/// +/// A sibling of `ai` on purpose: the `generate_image` tool resolves its target +/// here, so a session's text model never decides which image provider is called, +/// and image services never borrow the chat model shape. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct MediaConfig { + pub image_services: Vec, + /// [`ImageServiceConfig::id`] used by the `generate_image` tool when a call + /// does not name a service. + #[serde(skip_serializing_if = "Option::is_none")] + pub default_image_service: Option, +} + +impl MediaConfig { + pub fn enabled_image_service(&self, id: &str) -> Option<&ImageServiceConfig> { + self.image_services + .iter() + .find(|service| service.id == id && service.enabled) + } + + /// Default service, or the only enabled service when no default was set. + pub fn effective_image_service(&self) -> Option<&ImageServiceConfig> { + if let Some(service) = self + .default_image_service + .as_deref() + .and_then(|id| self.enabled_image_service(id)) + { + return Some(service); + } + + let mut enabled = self.image_services.iter().filter(|service| service.enabled); + let first = enabled.next()?; + enabled.next().is_none().then_some(first) + } +} + /// Model choice for a subagent created in the context of a parent session. /// /// `Inherit` is intentionally distinct from a model ID so a user-configured @@ -1955,6 +2068,7 @@ impl Default for GlobalConfig { terminal: TerminalConfig::default(), workspace: WorkspaceConfig::default(), ai: AIConfig::default(), + media: MediaConfig::default(), memories: MemoriesConfig::default(), project: ProjectConfig::default(), tool_permissions: ToolPermissionConfig::default(), @@ -4447,3 +4561,115 @@ mod external_session_discovery_tests { assert_eq!(serde_json::to_value(config).unwrap(), legacy); } } + +#[cfg(test)] +mod media_config_tests { + use super::*; + + fn service(id: &str, enabled: bool) -> ImageServiceConfig { + ImageServiceConfig { + id: id.to_string(), + name: id.to_string(), + dialect: ImageDialect::DashScopeNative, + base_url: "https://dashscope.aliyuncs.com/api/v1".to_string(), + model_name: "qwen-image-2.0".to_string(), + api_key: "sk-test".to_string(), + enabled, + params: ImageGenerationParams { + size: Some("2048*2048".to_string()), + watermark: Some(false), + ..ImageGenerationParams::default() + }, + ..ImageServiceConfig::default() + } + } + + #[test] + fn documents_written_before_media_existed_load_with_an_empty_section() { + let mut legacy = serde_json::to_value(GlobalConfig::default()).unwrap(); + legacy.as_object_mut().unwrap().remove("media"); + + let config: GlobalConfig = serde_json::from_value(legacy).unwrap(); + + assert!(config.media.image_services.is_empty()); + assert_eq!(config.media.default_image_service, None); + assert!(config.media.effective_image_service().is_none()); + } + + #[test] + fn media_section_is_always_serialized_so_the_dot_path_getter_can_read_it() { + let value = serde_json::to_value(GlobalConfig::default()).unwrap(); + + assert_eq!( + value.get("media"), + Some(&serde_json::json!({ "image_services": [] })) + ); + } + + #[test] + fn image_service_round_trips_and_ignores_unknown_future_fields() { + let media = MediaConfig { + image_services: vec![service("svc-a", true)], + default_image_service: Some("svc-a".to_string()), + }; + let mut value = serde_json::to_value(&media).unwrap(); + value["image_services"][0]["future_field"] = serde_json::json!("ignored"); + + let parsed: MediaConfig = serde_json::from_value(value).unwrap(); + + assert_eq!(parsed, media); + assert_eq!( + parsed.image_services[0].dialect, + ImageDialect::DashScopeNative + ); + } + + #[test] + fn dialect_wire_names_are_stable() { + for (dialect, wire) in [ + (ImageDialect::OpenAiImages, "openai_images"), + (ImageDialect::DashScopeNative, "dashscope_native"), + ] { + assert_eq!(serde_json::to_value(dialect).unwrap(), wire); + assert_eq!( + serde_json::from_value::(wire.into()).unwrap(), + dialect + ); + } + } + + #[test] + fn effective_service_prefers_the_default_pointer() { + let media = MediaConfig { + image_services: vec![service("svc-a", true), service("svc-b", true)], + default_image_service: Some("svc-b".to_string()), + }; + + assert_eq!(media.effective_image_service().unwrap().id, "svc-b"); + } + + #[test] + fn effective_service_falls_back_only_when_exactly_one_is_enabled() { + let single = MediaConfig { + image_services: vec![service("svc-a", true), service("svc-b", false)], + default_image_service: None, + }; + assert_eq!(single.effective_image_service().unwrap().id, "svc-a"); + + let ambiguous = MediaConfig { + image_services: vec![service("svc-a", true), service("svc-b", true)], + default_image_service: None, + }; + assert!(ambiguous.effective_image_service().is_none()); + } + + #[test] + fn a_disabled_default_pointer_is_not_served() { + let media = MediaConfig { + image_services: vec![service("svc-a", false)], + default_image_service: Some("svc-a".to_string()), + }; + + assert!(media.effective_image_service().is_none()); + } +} diff --git a/src/crates/contracts/core-types/src/ai.rs b/src/crates/contracts/core-types/src/ai.rs index 78a25dd03a..26494362e4 100644 --- a/src/crates/contracts/core-types/src/ai.rs +++ b/src/crates/contracts/core-types/src/ai.rs @@ -898,6 +898,61 @@ pub struct RemoteModelRouting { pub request_url: String, } +/// Upper bound accepted for a single image generation request. Dialects with a +/// tighter provider limit clamp further inside the adapter. +pub const MAX_IMAGES_PER_GENERATION_REQUEST: u32 = 10; + +/// Wire protocol used by an image generation provider. +/// +/// Endpoint path, request body shape, size syntax and response shape all vary +/// per provider, so this is the single fact that drives request construction +/// and response parsing. It is never inferred from a base URL. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum ImageDialect { + /// OpenAI Images: `POST {base}/images/generations`, JSON body, `data[]` result. + #[default] + #[serde(rename = "openai_images")] + OpenAiImages, + /// Alibaba DashScope native Qwen-Image: + /// `POST {base}/services/aigc/multimodal-generation/generation`. + #[serde(rename = "dashscope_native")] + DashScopeNative, + /// ChatGPT subscription images, driven through Codex CLI's ChatGPT backend: + /// `POST {base}/responses` with an `image_generation` tool, streamed back as + /// SSE `response.output_item.done` items. This is a separate billing and + /// protocol path from [`Self::OpenAiImages`]: it takes the OAuth access token + /// from a Codex subscription login and never reaches the platform image API. + #[serde(rename = "chatgpt_codex")] + ChatgptCodex, +} + +impl ImageDialect { + /// Endpoint path appended to the service base URL for this dialect. + pub fn endpoint_path(self) -> &'static str { + match self { + Self::OpenAiImages => "/images/generations", + Self::DashScopeNative => "/services/aigc/multimodal-generation/generation", + Self::ChatgptCodex => "/responses", + } + } + + /// Provider-specific cap for one request. + pub fn max_images_per_request(self) -> u32 { + match self { + Self::OpenAiImages => MAX_IMAGES_PER_GENERATION_REQUEST, + Self::DashScopeNative => 6, + // The subscription route returns one image per tool call. + Self::ChatgptCodex => 1, + } + } + + /// Whether this dialect authenticates with a subscription login rather than + /// a provider API key. + pub fn requires_subscription(self) -> bool { + matches!(self, Self::ChatgptCodex) + } +} + #[cfg(test)] mod remote_model_routing_compatibility { #[test] diff --git a/src/crates/contracts/core-types/src/lib.rs b/src/crates/contracts/core-types/src/lib.rs index 89ae3e9cde..51c66a93dd 100644 --- a/src/crates/contracts/core-types/src/lib.rs +++ b/src/crates/contracts/core-types/src/lib.rs @@ -22,19 +22,19 @@ pub mod tool_presentation; pub mod worktree; pub use ai::{ - AIConfig, ConnectionTestMessageCode, ConnectionTestResult, Message, ModelAccountOrigin, - ModelReasoningSummaryPart, ModelRequestContext, ModelResponseReplay, ModelResponseReplayItem, - ModelsDevCatalogSource, ModelsDevCatalogStatus, ModelsDevReasoningCatalog, - ModelsDevReasoningModel, ModelsDevReasoningProvider, ModelsDevRefreshResult, - ModelsDevRefreshStatus, ProviderCatalog, ProviderCatalogEndpoint, ProviderCatalogModel, - ProviderCatalogModelCapabilities, ProviderCatalogModelLimits, ProviderCatalogModelPricing, - ProviderCatalogModelSource, ProviderCatalogProvider, ProviderCatalogSource, - ProviderCatalogUpstreamProvider, ProxyConfig, ReasoningCapabilityStatus, + AIConfig, ConnectionTestMessageCode, ConnectionTestResult, ImageDialect, Message, + ModelAccountOrigin, ModelReasoningSummaryPart, ModelRequestContext, ModelResponseReplay, + ModelResponseReplayItem, ModelsDevCatalogSource, ModelsDevCatalogStatus, + ModelsDevReasoningCatalog, ModelsDevReasoningModel, ModelsDevReasoningProvider, + ModelsDevRefreshResult, ModelsDevRefreshStatus, ProviderCatalog, ProviderCatalogEndpoint, + ProviderCatalogModel, ProviderCatalogModelCapabilities, ProviderCatalogModelLimits, + ProviderCatalogModelPricing, ProviderCatalogModelSource, ProviderCatalogProvider, + ProviderCatalogSource, ProviderCatalogUpstreamProvider, ProxyConfig, ReasoningCapabilityStatus, ReasoningCatalogBinding, ReasoningCatalogProjection, ReasoningCatalogProjectionRequest, ReasoningConfig, ReasoningContentKind, ReasoningPreset, ReasoningPresetAction, ReasoningPresetDescriptor, ReasoningPresetSource, RemoteModelInfo, RemoteModelRouting, ToolCall, ToolCallConfirmationDetails, ToolCallRequestInfo, ToolCallResponseInfo, - ToolDefinition, + ToolDefinition, MAX_IMAGES_PER_GENERATION_REQUEST, }; pub use errors::{AiErrorDetail, ErrorCategory}; pub use model::{ diff --git a/src/crates/contracts/product-domains/src/remote_surface/table.rs b/src/crates/contracts/product-domains/src/remote_surface/table.rs index 2963eab2ba..0cb14f01b2 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/table.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/table.rs @@ -774,6 +774,7 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("terminal_write", Routed, Proxied, HANDLED), op("test_ai_config_connection", Unaudited, Proxied, HANDLED), op("test_ai_connection", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), + op("test_image_generation_service", Agnostic, Proxied, CLI_NOT_IMPLEMENTED), op("toggle_main_window_fullscreen", LocalOnly, ControllerLocal, REFUSED), op("touch_session_activity", Unaudited, Proxied, HANDLED), op("unarchive_session", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), diff --git a/src/crates/execution/tool-provider-groups/src/lib.rs b/src/crates/execution/tool-provider-groups/src/lib.rs index 1caf1f92b8..ca219da096 100644 --- a/src/crates/execution/tool-provider-groups/src/lib.rs +++ b/src/crates/execution/tool-provider-groups/src/lib.rs @@ -101,7 +101,9 @@ pub fn tool_feature_group(tool_name: &str) -> Option { } "WebSearch" | "WebFetch" | "ControlHub" => Some(ToolPackFeatureGroup::BrowserWeb), "ComputerUse" => Some(ToolPackFeatureGroup::ComputerUse), - "view_image" | "analyze_image" => Some(ToolPackFeatureGroup::ImageAnalysis), + "view_image" | "analyze_image" | "generate_image" => { + Some(ToolPackFeatureGroup::ImageAnalysis) + } "GenerativeUI" | "InitMiniApp" | "FinalizeMiniApp" | "PublishMiniApp" | "PublishAppearance" | "Playbook" => Some(ToolPackFeatureGroup::MiniApp), "PageDeploy" | "PagePublish" => Some(ToolPackFeatureGroup::Pages), @@ -177,6 +179,7 @@ const PRODUCT_TOOL_PROVIDER_GROUP_PLAN: &[ToolProviderGroupPlan] = &[ "Read", "view_image", "analyze_image", + "generate_image", "Glob", "Grep", "Write", diff --git a/src/crates/services/services-core/Cargo.toml b/src/crates/services/services-core/Cargo.toml index 1a54c638b1..e11052106a 100644 --- a/src/crates/services/services-core/Cargo.toml +++ b/src/crates/services/services-core/Cargo.toml @@ -63,6 +63,7 @@ routing-state = ["json-io", "dep:sha2"] resource-capacity = ["json-io", "dep:sha2", "dep:uuid"] session-event-format = ["dep:openbitfun-events"] tls-provider = ["dep:rustls"] +system-proxy = ["dep:windows", "windows/Win32_Foundation", "windows/Win32_System_Registry"] credential-vault = [ "dep:aes-gcm", "dep:anyhow", diff --git a/src/crates/services/services-core/src/lib.rs b/src/crates/services/services-core/src/lib.rs index 921f27f655..d17bdcd365 100644 --- a/src/crates/services/services-core/src/lib.rs +++ b/src/crates/services/services-core/src/lib.rs @@ -86,6 +86,8 @@ pub mod session_usage; pub mod storage_cleanup; #[cfg(feature = "process-runtime")] pub mod system; +#[cfg(feature = "system-proxy")] +pub mod system_proxy; #[cfg(feature = "tls-provider")] pub mod tls_provider; #[cfg(feature = "local-storage")] diff --git a/src/crates/services/services-core/src/system_proxy.rs b/src/crates/services/services-core/src/system_proxy.rs new file mode 100644 index 0000000000..54118afc96 --- /dev/null +++ b/src/crates/services/services-core/src/system_proxy.rs @@ -0,0 +1,262 @@ +//! Best-effort detection of the operating system HTTP proxy. +//! +//! Desktop users often reach an AI provider only through a local proxy agent +//! (Clash, V2Ray, a corporate gateway) that is configured in the OS. reqwest +//! reads only process environment variables, so a client built without an +//! explicit proxy bypasses that setting and fails to connect even though the +//! browser and other apps work. This module supplies the missing OS view. +//! +//! Detection never fails the caller: an unreadable or malformed setting returns +//! `None` and the request proceeds without a proxy. + +/// Proxy the operating system would apply to an HTTPS request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SystemProxy { + /// Proxy URL with an explicit scheme, for example `http://127.0.0.1:7897`. + pub url: String, + /// Hosts that must bypass the proxy, ready for `reqwest::NoProxy` syntax. + pub no_proxy: Option, +} + +/// Hosts that must never be sent through a system proxy. +/// +/// Local model servers (Ollama, LM Studio, llama.cpp) bind to loopback and a +/// proxy agent is not required for them; routing those through the proxy is a +/// common source of hard-to-diagnose connection failures. +#[cfg(any(windows, test))] +const LOOPBACK_NO_PROXY: &str = "localhost,127.0.0.1,::1"; + +/// Detect the proxy the current user configured in the operating system. +#[cfg(windows)] +pub fn detect() -> Option { + windows_registry::detect() +} + +/// Environment variables remain the only portable source outside Windows, and +/// reqwest already honours them on its own. +#[cfg(not(windows))] +pub fn detect() -> Option { + None +} + +/// Turn a registry `ProxyServer` value into a single proxy URL. +/// +/// The setting is either a bare `host:port` applied to every protocol or a +/// per-protocol list such as `http=127.0.0.1:7897;https=127.0.0.1:7897`. +#[cfg(any(windows, test))] +fn parse_proxy_server(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + + let candidate = if trimmed.contains('=') { + let mut http = None; + let mut https = None; + for entry in trimmed.split(';') { + let mut pair = entry.splitn(2, '='); + let scheme = pair.next().unwrap_or_default().trim().to_ascii_lowercase(); + let value = pair.next().unwrap_or_default().trim(); + if value.is_empty() { + continue; + } + match scheme.as_str() { + "https" => https = Some(value.to_string()), + "http" => http = Some(value.to_string()), + _ => {} + } + } + https.or(http)? + } else { + trimmed.to_string() + }; + + Some(with_scheme(&candidate)) +} + +#[cfg(any(windows, test))] +fn with_scheme(value: &str) -> String { + if value.contains("://") { + value.to_string() + } else { + format!("http://{value}") + } +} + +/// Convert a registry `ProxyOverride` value into `reqwest::NoProxy` syntax. +/// +/// `` means "bypass plain hostnames"; entries are separated by `;` and +/// may use `*` wildcards, which reqwest understands. +#[cfg(any(windows, test))] +fn parse_proxy_override(raw: &str) -> Option { + let mut entries: Vec = Vec::new(); + let mut bypass_plain_hostnames = false; + + for entry in raw.split(';') { + let entry = entry.trim(); + if entry.is_empty() { + continue; + } + if entry.eq_ignore_ascii_case("") { + bypass_plain_hostnames = true; + continue; + } + entries.push(entry.to_string()); + } + + if bypass_plain_hostnames && !entries.iter().any(|entry| entry == "*") { + entries.push("*".to_string()); + } + + (!entries.is_empty()).then(|| entries.join(",")) +} + +/// Combine the mandatory loopback bypass with the user's own override list. +#[cfg(any(windows, test))] +fn merged_no_proxy(override_list: Option) -> Option { + match override_list { + Some(list) if !list.is_empty() => Some(format!("{LOOPBACK_NO_PROXY},{list}")), + _ => Some(LOOPBACK_NO_PROXY.to_string()), + } +} + +#[cfg(windows)] +mod windows_registry { + use super::{merged_no_proxy, parse_proxy_override, parse_proxy_server, SystemProxy}; + use windows::{ + core::{w, PCWSTR}, + Win32::{ + Foundation::ERROR_SUCCESS, + System::Registry::{RegGetValueW, HKEY_CURRENT_USER, RRF_RT_REG_DWORD, RRF_RT_REG_SZ}, + }, + }; + + const INTERNET_SETTINGS: PCWSTR = + w!("Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings"); + + pub(super) fn detect() -> Option { + if read_dword(w!("ProxyEnable"))? == 0 { + return None; + } + + let server = read_string(w!("ProxyServer"))?; + let url = parse_proxy_server(&server)?; + let no_proxy = merged_no_proxy( + read_string(w!("ProxyOverride")).and_then(|raw| parse_proxy_override(&raw)), + ); + + Some(SystemProxy { url, no_proxy }) + } + + fn read_dword(name: PCWSTR) -> Option { + let mut value = 0u32; + let mut size = std::mem::size_of::() as u32; + let status = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + INTERNET_SETTINGS, + name, + RRF_RT_REG_DWORD, + None, + Some((&mut value as *mut u32).cast()), + Some(&mut size), + ) + }; + (status == ERROR_SUCCESS).then_some(value) + } + + fn read_string(name: PCWSTR) -> Option { + let mut stack = [0u16; 1024]; + let mut size = (stack.len() * 2) as u32; + let status = unsafe { + RegGetValueW( + HKEY_CURRENT_USER, + INTERNET_SETTINGS, + name, + RRF_RT_REG_SZ, + None, + Some(stack.as_mut_ptr().cast()), + Some(&mut size), + ) + }; + if status != ERROR_SUCCESS { + return None; + } + + let written = (size as usize / 2).min(stack.len()); + let end = stack[..written] + .iter() + .position(|unit| *unit == 0) + .unwrap_or(written); + Some(String::from_utf16_lossy(&stack[..end])) + } +} + +#[cfg(test)] +mod tests { + use super::{merged_no_proxy, parse_proxy_override, parse_proxy_server}; + + #[test] + fn bare_host_and_port_gets_a_scheme() { + assert_eq!( + parse_proxy_server("127.0.0.1:7897").as_deref(), + Some("http://127.0.0.1:7897") + ); + } + + #[test] + fn explicit_scheme_is_preserved() { + assert_eq!( + parse_proxy_server("socks5://127.0.0.1:1080").as_deref(), + Some("socks5://127.0.0.1:1080") + ); + } + + #[test] + fn per_protocol_list_prefers_https() { + assert_eq!( + parse_proxy_server("http=127.0.0.1:7890;https=127.0.0.1:7891").as_deref(), + Some("http://127.0.0.1:7891") + ); + assert_eq!( + parse_proxy_server("http=127.0.0.1:7890").as_deref(), + Some("http://127.0.0.1:7890") + ); + } + + #[test] + fn empty_server_value_is_not_a_proxy() { + assert_eq!(parse_proxy_server(" "), None); + assert_eq!(parse_proxy_server("http=;https="), None); + } + + #[test] + fn override_list_keeps_explicit_entries() { + assert_eq!( + parse_proxy_override("*.internal;10.*").as_deref(), + Some("*.internal,10.*") + ); + } + + #[test] + fn override_list_translates_local_marker() { + assert_eq!(parse_proxy_override("").as_deref(), Some("*")); + assert_eq!( + parse_proxy_override(";*.corp.example").as_deref(), + Some("*.corp.example,*") + ); + assert_eq!(parse_proxy_override(" "), None); + } + + #[test] + fn loopback_always_bypasses_the_proxy() { + assert_eq!( + merged_no_proxy(None).as_deref(), + Some("localhost,127.0.0.1,::1") + ); + assert_eq!( + merged_no_proxy(Some("*.corp".to_string())).as_deref(), + Some("localhost,127.0.0.1,::1,*.corp") + ); + } +} diff --git a/src/web-ui/src/app/scenes/settings/pages/ai/ModelPoolCardActions.tsx b/src/web-ui/src/app/scenes/settings/pages/ai/ModelPoolCardActions.tsx index cbe28c89d5..95a9ca0358 100644 --- a/src/web-ui/src/app/scenes/settings/pages/ai/ModelPoolCardActions.tsx +++ b/src/web-ui/src/app/scenes/settings/pages/ai/ModelPoolCardActions.tsx @@ -4,6 +4,8 @@ import { Icon, IconButton, MenuPopover, Switch, Tooltip, type MenuEntry } from ' import { useI18n } from '@/infrastructure/i18n'; interface ModelPoolCardActionsProps { + disabled?: boolean; + testLabel?: string; modelLabel: string; enabled: boolean; isTesting: boolean; @@ -15,6 +17,8 @@ interface ModelPoolCardActionsProps { } export function ModelPoolCardActions({ + disabled = false, + testLabel: explicitTestLabel, modelLabel, enabled, isTesting, @@ -31,7 +35,7 @@ export function ModelPoolCardActions({ const moreLabel = t('pool.moreActions', { model: modelLabel }); const testLabel = !connectionTestSupported ? t('messages.testUnsupportedOnHost') - : isTesting ? t('messages.testing') : t('actions.test'); + : isTesting ? t('messages.testing') : explicitTestLabel ?? t('actions.test'); const menuItems: MenuEntry[] = [ { id: 'delete', label: t('actions.delete'), icon: , tone: 'danger', onSelect: onDelete }, ]; @@ -51,7 +55,7 @@ export function ModelPoolCardActions({ variant="quiet" aria-label={testLabel} loading={isTesting} - disabled={!connectionTestSupported} + disabled={disabled || !connectionTestSupported} onClick={onTest} icon={} /> @@ -61,6 +65,7 @@ export function ModelPoolCardActions({ size="sm" variant="quiet" aria-label={t('actions.edit')} + disabled={disabled} onClick={onEdit} icon={} /> @@ -71,6 +76,7 @@ export function ModelPoolCardActions({ size="sm" variant="quiet" aria-label={moreLabel} + disabled={disabled} aria-haspopup="menu" aria-expanded={menuOpen} aria-controls={menuOpen ? menuId : undefined} @@ -88,6 +94,7 @@ export function ModelPoolCardActions({ placement="bottom" /> onEnabledChange(event.target.checked)} diff --git a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx index 8cf0c4094d..91291e51c5 100644 --- a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx +++ b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.tsx @@ -28,6 +28,9 @@ import { ConfigRetryState, } from '@/infrastructure/config/components/common'; import DefaultModelConfig from '@/infrastructure/config/components/DefaultModelConfig'; +import { ImageGenerationAcquisitionRow, ImageGenerationDefaultSlot, ImageGenerationDialogs, ImageGenerationModelCard } from '@/infrastructure/config/components/image-generation/ImageGenerationSection'; +import { useImageGenerationSettings } from '@/infrastructure/config/components/image-generation/useImageGenerationSettings'; +import { matchesImageModel } from '@/infrastructure/config/components/image-generation/imageModels'; import ModelTagsField from '@/infrastructure/config/components/ModelTagsField'; import { configsNeedingAutoTest, @@ -434,6 +437,7 @@ const ModelSettingsPage: React.FC = () => { const { t, i18n } = useI18n('settings/models'); const { t: tDefault } = useI18n('settings/default-model'); const { t: tComponents } = useI18n('components'); + const { t: tMedia } = useI18n('settings/media'); const subscriptionName = useCallback((account: SubscriptionAccount): string => { const labels: Record = { codex: t('subscriptionAuth.products.codex'), @@ -468,9 +472,10 @@ const ModelSettingsPage: React.FC = () => { const initialIdle = timeoutSeed.get('ai.stream_idle_timeout_secs', null); const initialTtft = timeoutSeed.get('ai.stream_ttft_timeout_secs', null); const [aiModels, setAiModels] = useState(modelSeed.get('ai.models', null) ?? []); + const imageSettings = useImageGenerationSettings(); const [poolDefaults, setPoolDefaults] = useState(modelSeed.get('ai.default_models', null) ?? {}); const [poolQuery, setPoolQuery] = useState(''); - const [poolCapability, setPoolCapability] = useState(''); + const [poolCapability, setPoolCapability] = useState(''); const [poolProvider, setPoolProvider] = useState(''); const [showTagSyncNotice, setShowTagSyncNotice] = useState(false); const [showSubscriptionManager, setShowSubscriptionManager] = useState(false); @@ -2651,9 +2656,10 @@ const ModelSettingsPage: React.FC = () => { : undefined; providers.set(getPoolProviderKey(model), label || getProviderDisplayName(model)); }); + imageSettings.media.image_services.forEach(service => providers.set(`image:${service.dialect}`, tMedia(`dialect.${service.dialect === 'chatgpt_codex' ? 'chatgptCodex' : service.dialect === 'dashscope_native' ? 'dashscopeNative' : 'openaiImages'}.name`))); return [...providers].map(([value, label]) => ({ value, label })) .sort((a, b) => a.label.localeCompare(b.label)); - }, [aiModels, importedSubscriptions]); + }, [aiModels, importedSubscriptions, imageSettings.media.image_services, tMedia]); const subscriptionModelCount = aiModels.filter(model => !!subscriptionProviderOf(model.auth)).length; const visiblePoolModels = useMemo(() => { const query = poolQuery.trim().toLowerCase(); @@ -2661,6 +2667,7 @@ const ModelSettingsPage: React.FC = () => { model.id === poolDefaults.primary ? 0 : model.id === poolDefaults.fast ? 1 : 2 ); return aiModels.filter(model => { + if (poolCapability === 'image_generation') return false; if (poolCapability && !getEffectiveModelCapabilities(model).includes(poolCapability)) return false; if (poolProvider && getPoolProviderKey(model) !== poolProvider) return false; const labels = poolRoles.filter(role => role.modelId && role.modelId === model.id).map(role => role.label); @@ -2668,6 +2675,7 @@ const ModelSettingsPage: React.FC = () => { .join(' ').toLowerCase().includes(query); }).sort((a, b) => priority(a) - priority(b)); }, [aiModels, poolQuery, poolCapability, poolProvider, poolRoles, poolDefaults]); + const visibleImageModels = imageSettings.media.image_services.filter(service => matchesImageModel(service, poolQuery, poolCapability, poolProvider)); if (creationMode === 'selection') { return ( @@ -4723,6 +4731,7 @@ const ModelSettingsPage: React.FC = () => { : !subscriptionLoadError && {t('acquisition.noneImported')}} + @@ -4767,6 +4776,7 @@ const ModelSettingsPage: React.FC = () => { )}>
} loading={isConfigLoading} disabled={configLoadError} onDefaultModelsChange={setPoolDefaults} />
{hasSuspendedEditorDraft && ( @@ -4779,11 +4789,12 @@ const ModelSettingsPage: React.FC = () => { onChange={event => setPoolQuery(event.target.value)} placeholder={t('pool.search')} aria-label={t('pool.search')} /> patch({ model_name: value })} /> + + + patch({ base_url: value })} /> + + + patch({ api_key: value })} /> + + } + +
+ patch({ name: value })} disabled={saving || testing} /> + {subscription && + patch({ model_name: value })} disabled={saving || testing} placeholder={t('wizard.fields.modelName.subscriptionPlaceholder')} /> + } + + ({ label: value, value }))]} + disabled={saving || testing} onCreateValue={value => value.trim() || undefined} + onValueChange={value => patchParams({ size: String(value) || undefined })} /> + + {preset.maxImagesPerRequest > 1 && + patchParams({ n })} disabled={saving || testing} /> + } + {preset.supports.quality && + ({ label: value, value }))} + onValueChange={value => patchParams({ quality: String(value) })} disabled={saving || testing} /> + } + {preset.supports.background && + ({ label: value, value }))} + onValueChange={value => patchParams({ background: String(value) })} disabled={saving || testing} /> + } + {preset.supports.negativePrompt && + patchParams({ negative_prompt })} disabled={saving || testing} /> + } + {preset.supports.watermark && + patchParams({ watermark: event.target.checked })} disabled={saving || testing} /> + } + {preset.supports.promptExtend && + patchParams({ prompt_extend: event.target.checked })} disabled={saving || testing} /> + } + {!subscription && + patch({ request_url })} disabled={saving || testing} /> + } + {!subscription &&

{resolveImageEndpoint(draft.base_url, draft.request_url, draft.dialect)}

} +
+ + {outcome &&
+

{outcome.success ? t('wizard.test.success', { time: formatNumber(outcome.elapsed_ms) }) : t(`wizard.errors.${outcome.error_kind || 'provider'}`)}

+ {outcome.message &&

{outcome.message}

} + {outcome.success && outcome.image_base64 && outcome.mime && {t('wizard.test.previewAlt')}} +
} + {duplicate &&

{t('wizard.validation.duplicate')}

} +

{t('wizard.test.billed')}

+ + + + + + + + ; +} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss new file mode 100644 index 0000000000..3be09a17d9 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss @@ -0,0 +1,39 @@ +.openbitfun-image-settings__editor, +.openbitfun-image-settings__advanced { + display: flex; + flex-direction: column; + gap: var(--openbitfun-space-3); + min-inline-size: 0; + font-family: var(--openbitfun-type-body-sm-font-family); + font-size: var(--openbitfun-type-body-sm-font-size); + font-weight: var(--openbitfun-type-body-sm-font-weight); + line-height: var(--openbitfun-type-body-sm-line-height); +} +.openbitfun-image-settings__hint { + margin: 0; + color: var(--openbitfun-color-content-secondary); +} +.openbitfun-image-settings__endpoint, +.openbitfun-image-settings__test-detail { + margin: 0; + overflow-wrap: anywhere; + color: var(--openbitfun-color-content-secondary); + font-family: var(--openbitfun-type-code-meta-font-family); + font-size: var(--openbitfun-type-code-meta-font-size); + line-height: var(--openbitfun-type-code-meta-line-height); +} +.openbitfun-image-settings__test-result { + display: flex; + flex-direction: column; + gap: var(--openbitfun-space-2); +} +.openbitfun-image-settings__test-result p { margin: 0; } +.openbitfun-image-settings__test-result[data-status='failure'] { + color: var(--openbitfun-color-status-danger-content); +} +.openbitfun-image-settings__test-image { + max-inline-size: 100%; + max-block-size: 320px; + object-fit: contain; + border-radius: var(--openbitfun-radius-sm); +} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.tsx b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.tsx new file mode 100644 index 0000000000..62d55691a7 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.tsx @@ -0,0 +1,90 @@ +import './ImageGenerationSection.scss'; +import { Button, Capsule, Card, CardFooter, CardHeader, Combobox, Icon, IconButton, OverflowText, StatusPill, Tooltip } from '@openbitfun/ui'; +import { ConfigPageRow, ConfigRetryState } from '../common'; +import { useI18n } from '@/infrastructure/i18n'; +import type { ReactNode } from 'react'; +import ImageGenerationEditor from './ImageGenerationEditor'; +import type { ImageGenerationSettings } from './useImageGenerationSettings'; +import type { ImageServiceConfig } from '../../types'; +import { findImageDialectPreset } from './imageDialects'; +import { imageModelLabel } from './imageModels'; + +type Props = { settings: ImageGenerationSettings }; + +export function ImageGenerationAcquisitionRow({ settings }: Props) { + const { t } = useI18n('settings/media'); + const { t: tModels } = useI18n('settings/models'); + return +
{t('acquisition.label')} + {t('acquisition.description')}
+ + }> + {settings.loadError ? void settings.load()} /> + : settings.media.image_services.length > 0 ?
    + {settings.media.image_services.map(service =>
  • + + } + aria-label={t('acquisition.remove', { name: service.name || service.model_name })} disabled={settings.saving || Boolean(settings.testingId)} + onClick={() => void settings.remove(service)} /> + }> + + +
    • )} +
: + {settings.loading ? tModels('messages.loading') : t('acquisition.empty')} + } + ; +} + +export function ImageGenerationDefaultSlot({ settings }: Props) { + const { t } = useI18n('settings/media'); + const { t: tDefault } = useI18n('settings/default-model'); + const enabled = settings.media.image_services.filter(service => service.enabled); + const options = [ + ...enabled.map(service => ({ label: imageModelLabel(service), value: service.id })), + ...settings.configuredModels.map(model => ({ label: `${model.name || model.model_name} · ${model.model_name}`, value: `configured:${model.id}` })), + ]; + if (settings.selectedId && !options.some(option => option.value === settings.selectedId)) { + options.push({ label: t('default.unavailable'), value: settings.selectedId }); + } + return + {tDefault('optional.capabilities.image_generation.label')} + + } /> + + }> + void settings.selectDefault(String(Array.isArray(value) ? value[0] ?? '' : value))} /> + ; +} + +export function ImageGenerationModelCard({ settings, service, actions }: Props & { service: ImageServiceConfig; actions: ReactNode }) { + const { t } = useI18n('settings/media'); + const label = imageModelLabel(service); + return
+ + {label} +
{t('title')} + {settings.selectedId === service.id && {t('card.defaultBadge')}} +
+
} /> + + {t(findImageDialectPreset(service.dialect).labelKey)} + {actions} + + + ; +} + +export function ImageGenerationDialogs({ settings }: Props) { + return ; +} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx new file mode 100644 index 0000000000..4b08b07173 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx @@ -0,0 +1,323 @@ +// @vitest-environment jsdom +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { AIModelConfig, MediaConfig } from '../../types'; +import { activateSurface } from '@/infrastructure/peer-device/deviceSurface'; +import { getSettingsDraftSnapshot } from '../../settingsDraftRegistry'; +import { ImageGenerationAcquisitionRow, ImageGenerationDefaultSlot, ImageGenerationDialogs } from './ImageGenerationSection'; +import { useImageGenerationSettings, type ImageGenerationSettings } from './useImageGenerationSettings'; +import DefaultModelConfig from '../DefaultModelConfig'; +import { imageServiceFromSubscriptionModel } from './imageModels'; + +globalThis.IS_REACT_ACT_ENVIRONMENT = true; +const mocks = vi.hoisted(() => ({ get: vi.fn(), getModels: vi.fn(), update: vi.fn(), success: vi.fn(), error: vi.fn(), test: vi.fn() })); +vi.mock('../../services/ConfigManager', () => ({ configManager: { + hasCachedConfig: () => false, getCachedConfig: () => undefined, + getConfig: mocks.get, getOptionalConfig: mocks.getModels, updateConfig: mocks.update, watch: () => () => {}, +} })); +vi.mock('@/shared/notification-system', () => ({ useNotification: () => ({ success: mocks.success, error: mocks.error }), notificationService: { success: mocks.success, error: mocks.error } })); +vi.mock('../ModelSelectPresentation', () => ({ useModelSelectPresentation: () => ({ buildModelOption: (model: AIModelConfig) => ({ label: model.model_name, value: model.id }) }) })); +vi.mock('react-i18next', async original => ({ ...await original(), useTranslation: () => ({ t: (key: string) => key }) })); +vi.mock('@/infrastructure/api/service-api/MediaApi', () => ({ mediaAPI: { testImageGenerationService: mocks.test } })); +vi.mock('@/infrastructure/i18n', () => ({ useI18n: () => ({ t: (key: string) => key, formatNumber: String }) })); + +const subscription: AIModelConfig = { + id: 'configured-codex', name: 'My subscription', provider: 'openai', model_name: 'account-chat-model', + base_url: 'https://chatgpt.com/backend-api/codex', api_key: 'unused-chat-secret', enabled: true, + auth: { type: 'subscription', provider: 'codex' }, category: 'general_chat', capabilities: ['text_chat'], +}; +const legacy: MediaConfig = { image_services: [{ id: 'legacy', name: 'Saved image', dialect: 'openai_images', + base_url: 'https://images.example/v1', request_url: 'https://images.example/v1/images/generations', + model_name: 'image-model', api_key: 'saved-image-key', enabled: true, params: { size: '1024x1024', n: 1, seed: 42 }, + custom_headers: { 'X-Tenant': 'kept' }, auth: { type: 'api_key' } }], default_image_service: 'legacy' }; +const deferred = () => { let resolve!: (value: T) => void; const promise = new Promise(done => { resolve = done; }); return { promise, resolve }; }; + +describe('image models in model settings', () => { + let host: HTMLDivElement; + let root: Root; + let media: MediaConfig; + let settings: ImageGenerationSettings; + function Fixture({ models = [subscription] }: { models?: AIModelConfig[] }) { + settings = useImageGenerationSettings(); + return <>
+
{}} imageGenerationSlot={} />
+ ; + } + const render = async (models?: AIModelConfig[]) => act(async () => { if (models) mocks.getModels.mockResolvedValue(models); root.render(); }); + const click = async (element: HTMLElement) => act(async () => { element.focus(); element.click(); }); + const choose = async (testId: string, label: string) => { + const control = document.querySelector(`[data-testid="${testId}"]`)!; + await click(control.matches('button') ? control : control.querySelector('button[data-openbitfun-part="trigger"]')!); + const option = [...document.querySelectorAll('[role="option"]')].find(element => element.textContent?.includes(label)); + expect(option, `option ${label}`).toBeTruthy(); await click(option!); + }; + const button = (label: string) => [...document.querySelectorAll('button')].find(element => element.textContent === label)!; + + beforeEach(() => { + activateSurface('local'); + media = JSON.parse(JSON.stringify(legacy)); + mocks.get.mockReset().mockImplementation(async () => media); + mocks.getModels.mockReset().mockResolvedValue([subscription]); + mocks.update.mockReset().mockImplementation(async (_path, update) => { media = update(media); return media; }); + mocks.success.mockReset(); mocks.error.mockReset(); mocks.test.mockReset(); + host = document.createElement('div'); document.body.append(host); root = createRoot(host); + }); + afterEach(() => { act(() => root.unmount()); host.remove(); activateSurface('local'); }); + + it('selects an already configured subscription as the image default without changing chat defaults', async () => { + await render(); + await choose('image-service-default-select', 'account-chat-model'); + expect(mocks.update).toHaveBeenCalledTimes(1); + expect(mocks.update.mock.calls[0][0]).toBe('media'); + expect(media.default_image_service).toBe('configured-image:configured-codex'); + expect(media.image_services[0]).toEqual(legacy.image_services[0]); + expect(media.image_services[1]).toMatchObject({ dialect: 'chatgpt_codex', model_name: 'account-chat-model', api_key: '', auth: subscription.auth }); + expect(host.querySelector('[data-testid="acquisition"]')?.textContent).toContain('My subscription'); + }); + + it('keeps explicit accounts separate and excludes account pools from image imports', async () => { + const first: AIModelConfig = { ...subscription, auth: { type: 'subscription', provider: 'codex', account_id: 'account-a' } }; + const second: AIModelConfig = { ...subscription, id: 'second-account', auth: { type: 'subscription', provider: 'codex', account_id: 'account-b' } }; + const pool: AIModelConfig = { ...subscription, id: 'account-pool', auth: { type: 'subscription_pool', provider: 'codex', accounts: ['account-a', 'account-b'], revision: 1 } }; + await render([first, second, pool]); + expect(settings.credentials).toHaveLength(2); + expect(settings.configuredModels).toHaveLength(2); + await act(async () => { await settings.selectDefault('configured:configured-codex'); }); + expect(media.image_services.at(-1)?.auth).toEqual(first.auth); + expect(settings.configuredModels.map(model => model.id)).toEqual(['second-account']); + }); + + it('keeps a persisted image account pool visible but disables unsupported editing and generation', async () => { + media = { ...legacy, image_services: [{ ...legacy.image_services[0], auth: { type: 'subscription_pool', provider: 'codex', accounts: ['account-a'], revision: 1 } }] }; + await render(); + await click([...host.querySelectorAll('button')].find(element => element.textContent?.includes('Saved image'))!); + expect(button('actions.save').disabled).toBe(true); + expect(button('wizard.test.run').disabled).toBe(true); + expect(button('actions.save').title).toBe('wizard.validation.unsupportedAuth'); + expect(media.image_services[0].auth?.type).toBe('subscription_pool'); + }); + + it('imports a subscription using the same picker flow without showing duplicate connection fields', async () => { + await render(); await click(host.querySelector('[data-testid="image-service-add"]')!); + await choose('image-service-source', 'My subscription'); + expect(document.querySelector('[role="dialog"] input[type="password"]')).toBeNull(); + expect(button('advanced')?.getAttribute('aria-expanded')).toBe('false'); + await click(button('actions.save')); + expect(media.image_services.at(-1)).toMatchObject({ name: 'My subscription', dialect: 'chatgpt_codex', model_name: '', api_key: '' }); + expect(document.querySelector('[role="dialog"]:not([aria-hidden="true"])')).toBeNull(); + expect(mocks.test).not.toHaveBeenCalled(); + }); + + it('blocks repeated provider imports in the dialog and at save time', async () => { + await render(); + await click(host.querySelector('[data-testid="image-service-add"]')!); + await choose('image-service-source', 'My subscription'); + await click(button('actions.save')); + const imported = media.image_services.at(-1)!; + await click(host.querySelector('[data-testid="image-service-add"]')!); + await choose('image-service-source', 'My subscription'); + expect(button('actions.save').disabled).toBe(true); + expect(document.querySelector('[role="dialog"] [role="alert"]')?.textContent).toBe('wizard.validation.duplicate'); + await act(async () => { expect(await settings.saveService({ ...imported, id: 'new-id', name: 'Renamed' })).toBe(false); }); + expect(media.image_services).toHaveLength(2); + expect(media.default_image_service).toBe('legacy'); + expect(mocks.error).toHaveBeenLastCalledWith('wizard.validation.duplicate'); + expect(mocks.test).not.toHaveBeenCalled(); + }); + + it('checks fresh persisted config when another window has already saved the same manual configuration', async () => { + await render(); + const original = media; + mocks.update.mockImplementationOnce(async (_path, update) => { + media = { ...media, image_services: [...media.image_services, { ...legacy.image_services[0], id: 'other-window', model_name: 'new-image' }] }; + media = update(media); + return media; + }); + await act(async () => { + expect(await settings.saveService({ ...legacy.image_services[0], id: 'new-draft', model_name: 'new-image' })).toBe(false); + }); + expect(media.image_services.map(service => service.id)).toEqual(['legacy', 'other-window']); + expect(media.default_image_service).toBe(original.default_image_service); + expect(mocks.success).not.toHaveBeenCalled(); + expect(mocks.error).toHaveBeenCalledWith('wizard.validation.duplicate'); + }); + + it('reuses an imported disabled service when its configured model becomes the default', async () => { + const imported = { ...imageServiceFromSubscriptionModel(subscription), id: 'manually-imported', enabled: false }; + media = { ...legacy, image_services: [...legacy.image_services, imported] }; + await render(); + await choose('image-service-default-select', 'account-chat-model'); + expect(media.default_image_service).toBe('manually-imported'); + expect(media.image_services).toHaveLength(2); + expect(media.image_services.find(service => service.id === imported.id)).toMatchObject({ enabled: true }); + }); + + it('preserves existing advanced parameters and the open draft after a failed save', async () => { + await render(); await click([...host.querySelectorAll('button')].find(element => element.textContent?.includes('Saved image'))!); + const field = [...document.querySelectorAll('.openbitfun-config-page-row')].find(element => element.textContent?.includes('wizard.fields.modelName.label'))!.querySelector('input')!; + await act(async () => { Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call(field, 'updated-image-model'); field.dispatchEvent(new Event('input', { bubbles: true })); }); + mocks.update.mockRejectedValueOnce(new Error('host offline')); + await click(button('actions.save')); + expect(document.querySelector('[role="dialog"]:not([aria-hidden="true"])')).not.toBeNull(); + expect(field.value).toBe('updated-image-model'); + expect(getSettingsDraftSnapshot().resources.find(item => item.id === 'image-generation-editor')?.dirty).toBe(true); + expect(media).toEqual(legacy); + await click(button('actions.save')); + expect(media.image_services[0]).toMatchObject({ model_name: 'updated-image-model', params: legacy.image_services[0].params, custom_headers: legacy.image_services[0].custom_headers }); + expect(media.default_image_service).toBe('legacy'); + }); + + it('keeps an image-only configuration selectable when there are no chat models', async () => { + await render([]); + const control = host.querySelector('[data-testid="image-service-default-select"] button')!; + expect(control.disabled).toBe(false); + await choose('image-service-default-select', 'Saved image'); + expect(media.default_image_service).toBe('legacy'); + }); + + it('does not overwrite a configuration when its initial load failed', async () => { + mocks.get.mockRejectedValueOnce(new Error('host offline')); + await render(); + expect(host.querySelector('[data-testid="image-service-add"]')!.disabled).toBe(true); + await act(async () => { expect(await settings.saveService(legacy.image_services[0])).toBe(false); }); + expect(mocks.update).not.toHaveBeenCalled(); + }); + + it('persists the first enabled image default and lets a manual selection survive reloads', async () => { + media = { image_services: [...legacy.image_services, { ...legacy.image_services[0], id: 'second', model_name: 'second-image-model' }] }; + await render([]); + expect(media.default_image_service).toBe('legacy'); + expect(settings.selectedId).toBe('legacy'); + expect(mocks.update).toHaveBeenCalledTimes(1); + expect(mocks.success).not.toHaveBeenCalled(); + expect(host.textContent).not.toContain('default.ambiguous'); + expect(host.querySelector('[data-testid="image-service-default-select"]')?.textContent).toContain('Saved image'); + expect(host.querySelector('[aria-invalid="true"]')).toBeNull(); + await choose('image-service-default-select', 'second-image-model'); + expect(media.default_image_service).toBe('second'); + await act(async () => { await settings.load(); }); + expect(settings.selectedId).toBe('second'); + expect(mocks.update).toHaveBeenCalledTimes(2); + }); + + it('replaces an unavailable default and falls back when the chosen service is disabled', async () => { + media = { image_services: [{ ...legacy.image_services[0], id: 'disabled', enabled: false }, ...legacy.image_services, + { ...legacy.image_services[0], id: 'second' }], default_image_service: 'disabled' }; + await render([]); + expect(media.default_image_service).toBe('legacy'); + await act(async () => { await settings.toggle(media.image_services[1], false); }); + expect(media.default_image_service).toBe('second'); + expect(media.image_services).toHaveLength(3); + }); + + it('does not repeatedly retry a failed automatic default save and permits manual retry', async () => { + media = { image_services: legacy.image_services }; + mocks.update.mockRejectedValueOnce(new Error('host offline')); + await render([]); + await render([]); + expect(mocks.update).toHaveBeenCalledTimes(1); + expect(mocks.error).toHaveBeenCalledTimes(1); + await choose('image-service-default-select', 'Saved image'); + expect(media.default_image_service).toBe('legacy'); + expect(mocks.update).toHaveBeenCalledTimes(2); + }); + + it('keeps a default selected by another editor while initializing the default', async () => { + media = { image_services: [...legacy.image_services, { ...legacy.image_services[0], id: 'second' }] }; + mocks.update.mockImplementationOnce(async (_path, update) => { + media = update({ ...media, default_image_service: 'second' }); + return media; + }); + await render([]); + expect(media.default_image_service).toBe('second'); + expect(settings.selectedId).toBe('second'); + }); + + it('closes drafts and resets the list while the next execution host loads', async () => { + const pending = deferred(); + await render(); await click(host.querySelector('[data-testid="image-service-add"]')!); + mocks.get.mockImplementation(async () => pending.promise); + await act(async () => { activateSurface('peer-device'); }); + expect(document.querySelector('[role="dialog"]:not([aria-hidden="true"])')).toBeNull(); + await act(async () => { pending.resolve({ image_services: [] }); }); + expect(settings.media.image_services).toEqual([]); + expect(host.textContent).not.toContain('Saved image'); + }); + + it('imports provider connection details without using a chat model as the image model', async () => { + const apiModel: AIModelConfig = { ...subscription, id: 'api-image', name: 'Image gateway', model_name: 'my-image-model', + base_url: 'https://images.example/v1', request_url: 'https://images.example/v1/chat/completions', + auth: { type: 'api_key' }, api_key: 'image-secret', custom_headers: { 'X-Tenant': 'image-tenant' } }; + await render([apiModel]); await click(host.querySelector('[data-testid="image-service-add"]')!); + await choose('image-service-source', 'Image gateway'); await click(button('actions.save')); + expect(media.image_services.at(-1)).toMatchObject({ name: 'Image gateway', dialect: 'openai_images', model_name: 'gpt-image-1', + base_url: 'https://images.example/v1', api_key: 'image-secret', custom_headers: { 'X-Tenant': 'image-tenant' } }); + expect(media.image_services.at(-1)?.request_url).toBeUndefined(); + }); + + it('lists each provider instance once while keeping separate providers selectable', async () => { + await render([ + { ...subscription, metadata: { provider_instance_id: 'subscription-provider' } }, + { ...subscription, id: 'second-chat-model', model_name: 'other-chat-model', metadata: { provider_instance_id: 'subscription-provider' } }, + { ...subscription, id: 'another-provider', name: 'Other subscription', metadata: { provider_instance_id: 'other-provider' } }, + ]); + await click(host.querySelector('[data-testid="image-service-add"]')!); + await click(document.querySelector('[data-testid="image-service-source"] button[data-openbitfun-part="trigger"]')!); + const options = [...document.querySelectorAll('[role="option"]')]; + expect(options.map(option => option.textContent)).toEqual(['import.manual', 'My subscription', 'Other subscription']); + expect(options.some(option => option.textContent?.includes('chat-model'))).toBe(false); + expect(settings.credentials).toHaveLength(2); + }); + + it('groups legacy provider connections without mixing distinct keys or endpoints', async () => { + const apiModel: AIModelConfig = { ...subscription, auth: { type: 'api_key' }, name: 'Gateway - chat-one', model_name: 'chat-one' }; + await render([ + apiModel, { ...apiModel, id: 'second-model', name: 'Gateway/chat-two', model_name: 'chat-two', base_url: `${apiModel.base_url}/` }, + { ...apiModel, id: 'other-key', name: 'Other account', api_key: 'another-secret' }, + { ...apiModel, id: 'other-endpoint', name: 'Other endpoint', base_url: 'https://other.example/v1' }, + ]); + expect(settings.credentials.map(credential => credential.label)).toEqual(['Gateway', 'Other account', 'Other endpoint']); + expect(settings.credentials.map(credential => credential.id)).toEqual(['configured-codex', 'other-key', 'other-endpoint']); + }); + + it('ignores a completed read from the previous host', async () => { + const oldRead = deferred(); + mocks.get.mockReturnValueOnce(oldRead.promise); + await render(); + mocks.get.mockResolvedValue({ image_services: [] }); + await act(async () => { activateSurface('next-host'); }); + await act(async () => { oldRead.resolve(legacy); }); + expect(settings.media.image_services).toEqual([]); + expect(host.textContent).not.toContain('Saved image'); + }); + + it('does not apply an old host save result or success notification after activation changes', async () => { + await render(); + const oldSave = deferred(); + mocks.update.mockReturnValueOnce(oldSave.promise); + let save!: Promise; + await act(async () => { save = settings.selectDefault('configured:configured-codex'); }); + mocks.get.mockResolvedValue({ image_services: [] }); + await act(async () => { activateSurface('next-host'); }); + await act(async () => { oldSave.resolve(legacy); await save; }); + expect(settings.media.image_services).toEqual([]); + expect(mocks.success).not.toHaveBeenCalled(); + expect(mocks.update).toHaveBeenCalledTimes(1); + }); + + it('does not offer or copy the previous device credentials while the next device model list loads', async () => { + await render(); + const nextModels = deferred(); + mocks.getModels.mockReturnValueOnce(nextModels.promise); + mocks.get.mockResolvedValue({ image_services: [] }); + await act(async () => { activateSurface('next-host'); }); + expect(settings.credentials).toEqual([]); + await act(async () => { await settings.selectDefault('configured:configured-codex'); }); + expect(mocks.update).not.toHaveBeenCalled(); + await act(async () => { nextModels.resolve([]); }); + expect(settings.configuredModels).toEqual([]); + expect(settings.credentials).toEqual([]); + }); +}); diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.test.ts b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.test.ts new file mode 100644 index 0000000000..3ab56c3d87 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from 'vitest'; +import type { ImageServiceConfig } from '@/infrastructure/config/types'; +import { resolveImageEndpoint, withImageServiceAuth } from './imageDialects'; + +const legacyService: ImageServiceConfig = { + id: 'saved-service', name: 'Saved image service', dialect: 'openai_images', + base_url: 'https://api.openai.com/v1', request_url: 'https://api.openai.com/v1/images/generations', + model_name: 'gpt-image-1', api_key: '', enabled: true, + auth: { type: 'api_key' }, params: { size: '1024x1024', n: 3 }, +}; + +describe('image service authentication routing', () => { + it('preserves unsupported account-pool records instead of converting them to API keys', () => { + const auth = { type: 'subscription_pool', provider: 'codex', accounts: ['account-a'], revision: 1 } as const; + const saved = { ...legacyService, auth: { ...auth, accounts: [...auth.accounts] } }; + expect(withImageServiceAuth(saved, saved.auth)).toEqual(saved); + }); + it('moves a reused Codex subscription off the public image endpoint', () => { + const result = withImageServiceAuth(legacyService, { type: 'subscription', provider: 'codex' }); + expect(result).toMatchObject({ id: 'saved-service', dialect: 'chatgpt_codex', model_name: '', params: { n: 1 } }); + expect(resolveImageEndpoint(result.base_url, result.request_url, result.dialect)) + .toBe('https://chatgpt.com/backend-api/codex/responses'); + expect(legacyService.request_url).toBe('https://api.openai.com/v1/images/generations'); + }); + + it('recovers a serialized legacy subscription selection without changing the stored record', () => { + const saved = JSON.parse(JSON.stringify({ ...legacyService, auth: { type: 'subscription', provider: 'codex' } })); + expect(withImageServiceAuth(saved, saved.auth).dialect).toBe('chatgpt_codex'); + expect(saved.dialect).toBe('openai_images'); + }); + + it('drops the subscription endpoint when returning to an API key', () => { + const subscription = withImageServiceAuth(legacyService, { type: 'subscription', provider: 'codex' }); + const result = withImageServiceAuth({ ...subscription, request_url: 'https://chatgpt.com/backend-api/codex/responses' }, { type: 'api_key' }); + expect(result.dialect).toBe('openai_images'); + expect(resolveImageEndpoint(result.base_url, result.request_url, result.dialect)).toBe('https://api.openai.com/v1/images/generations'); + }); + + it('preserves custom API-key gateways and explicitly chosen subscription models', () => { + expect(withImageServiceAuth(legacyService, { type: 'api_key' })).toEqual(legacyService); + const auth = { type: 'subscription' as const, provider: 'codex' as const }; + const service = { ...withImageServiceAuth(legacyService, auth), model_name: 'account-model' }; + expect(withImageServiceAuth(service, auth).model_name).toBe('account-model'); + }); +}); diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts new file mode 100644 index 0000000000..1f467e5a06 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts @@ -0,0 +1,130 @@ +import type { AuthConfig, ImageDialect, ImageServiceConfig } from '@/infrastructure/config/types'; + +/** + * Per-dialect presentation facts. + * + * The backend owns the request and response shapes; this table only decides what + * the settings UI may offer, so an unsupported field is never shown as if it + * worked. + */ +export interface ImageDialectPreset { + id: ImageDialect; + labelKey: string; + descriptionKey: string; + /** Endpoint path appended to the service root. */ + endpointPath: string; + defaultBaseUrl: string; + defaultModel: string; + /** Common size values in this dialect's own syntax. */ + sizes: string[]; + maxImagesPerRequest: number; + supports: { + negativePrompt: boolean; + quality: boolean; + background: boolean; + outputFormat: boolean; + watermark: boolean; + promptExtend: boolean; + seed: boolean; + }; +} + +export const IMAGE_DIALECT_PRESETS: readonly ImageDialectPreset[] = [ + { + id: 'openai_images', + labelKey: 'dialect.openaiImages.name', + descriptionKey: 'dialect.openaiImages.description', + endpointPath: '/images/generations', + defaultBaseUrl: 'https://api.openai.com/v1', + defaultModel: 'gpt-image-1', + sizes: ['1024x1024', '1536x1024', '1024x1536'], + maxImagesPerRequest: 10, + supports: { + negativePrompt: false, + quality: true, + background: true, + outputFormat: false, + watermark: false, + promptExtend: false, + seed: false, + }, + }, + { + id: 'dashscope_native', + labelKey: 'dialect.dashscopeNative.name', + descriptionKey: 'dialect.dashscopeNative.description', + endpointPath: '/services/aigc/multimodal-generation/generation', + defaultBaseUrl: 'https://dashscope.aliyuncs.com/api/v1', + defaultModel: 'qwen-image-plus', + sizes: ['1024*1024', '1664*928', '928*1664', '2048*2048'], + maxImagesPerRequest: 6, + supports: { + negativePrompt: true, + quality: false, + background: false, + outputFormat: false, + watermark: true, + promptExtend: true, + seed: true, + }, + }, + { + id: 'chatgpt_codex', + labelKey: 'dialect.chatgptCodex.name', + descriptionKey: 'dialect.chatgptCodex.description', + endpointPath: '/responses', + // The login supplies this address; the value here only seeds a new draft. + defaultBaseUrl: 'https://chatgpt.com/backend-api/codex', + // Empty means "discover from the account catalog when sending". + defaultModel: '', + sizes: ['1024x1024', '1536x1024', '1024x1536'], + maxImagesPerRequest: 1, + supports: { + negativePrompt: false, + quality: true, + background: true, + outputFormat: false, + watermark: false, + promptExtend: false, + seed: false, + }, + }, +]; + +export function findImageDialectPreset(dialect: ImageDialect): ImageDialectPreset { + return IMAGE_DIALECT_PRESETS.find(preset => preset.id === dialect) ?? IMAGE_DIALECT_PRESETS[0]; +} + +/** Keep credential reuse and explicit authentication selection on the same route. */ +export function withImageServiceAuth(service: ImageServiceConfig, auth: AuthConfig): ImageServiceConfig { + if (auth.type === 'subscription_pool') return { ...service, auth }; + const dialect = auth.type === 'subscription' && auth.provider === 'codex' + ? 'chatgpt_codex' + : service.dialect === 'chatgpt_codex' ? 'openai_images' : service.dialect; + if (dialect === service.dialect) return { ...service, auth }; + const preset = findImageDialectPreset(dialect); + return { + ...service, + auth, + dialect, + base_url: preset.defaultBaseUrl, + request_url: undefined, + model_name: preset.defaultModel, + params: { ...service.params, size: preset.sizes[0], n: 1 }, + }; +} + +/** Endpoint the request will actually be sent to. */ +export function resolveImageEndpoint( + baseUrl: string, + requestUrl: string | undefined, + dialect: ImageDialect, +): string { + const override = requestUrl?.trim(); + if (override) return override; + + const root = baseUrl.trim().replace(/\/+$/, ''); + if (!root) return ''; + const path = findImageDialectPreset(dialect).endpointPath; + return root.endsWith(path) ? root : `${root}${path}`; +} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.test.ts b/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.test.ts new file mode 100644 index 0000000000..ed593e45c7 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest'; +import type { ImageServiceConfig } from '../../types'; +import { duplicateImageService, sameImageService } from './imageModels'; + +const service: ImageServiceConfig = { + id: 'saved', name: 'Images', dialect: 'openai_images', base_url: 'https://images.example/v1', + model_name: 'image-model', api_key: 'secret', enabled: true, params: { n: 1 }, +}; + +describe('image service identity', () => { + it('recognizes the same request regardless of id, name, enabled state, or explicit defaults', () => { + expect(sameImageService(service, { ...service, id: 'new', name: 'Renamed', enabled: false, + base_url: `${service.base_url}/`, request_url: 'https://images.example/v1/images/generations', + auth: { type: 'api_key' }, custom_headers: {}, custom_headers_mode: 'merge', skip_ssl_verify: false, + params: { size: undefined } })).toBe(true); + }); + + it.each>([ + { model_name: 'other-image-model' }, + { base_url: 'https://other.example/v1' }, + { api_key: 'different-account' }, + { params: { n: 2 } }, + { params: { size: '1536x1024' } }, + { params: { seed: 42 } }, + { custom_headers: { 'X-Tenant': 'another' } }, + { custom_headers_mode: 'replace' }, + { skip_ssl_verify: true }, + ])('keeps different request configurations separate (%j)', difference => { + expect(sameImageService(service, { ...service, ...difference })).toBe(false); + }); + + it('ignores property order for headers and advanced generation parameters', () => { + expect(sameImageService({ ...service, custom_headers: { a: 'a', b: 'b' }, params: { seed: 1, n: 1 } }, + { ...service, custom_headers: { b: 'b', a: 'a' }, params: { n: 1, seed: 1 } })).toBe(true); + }); + + it('recognizes subscription imports without comparing unused API keys', () => { + const subscription: ImageServiceConfig = { ...service, dialect: 'chatgpt_codex', model_name: '', + base_url: 'https://chatgpt.com/backend-api/codex', auth: { type: 'subscription', provider: 'codex' } }; + expect(duplicateImageService([subscription], { ...subscription, id: 'new', name: 'Codex', api_key: '' })).toEqual(subscription); + expect(sameImageService(subscription, { ...subscription, model_name: 'other-driver' })).toBe(false); + }); + + it('blocks an edit that collides with another record but allows unchanged legacy duplicates to be edited', () => { + const second = { ...service, id: 'second', model_name: 'other-model' }; + expect(duplicateImageService([service, second], { ...second, model_name: service.model_name })).toEqual(service); + expect(duplicateImageService([service, { ...service, id: 'legacy-duplicate' }], { ...service, name: 'Renamed' })).toBeUndefined(); + }); +}); diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.ts b/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.ts new file mode 100644 index 0000000000..d8ab895c7e --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/imageModels.ts @@ -0,0 +1,79 @@ +import type { AIModelConfig, ImageServiceConfig, MediaConfig } from '../../types'; +import { stableJson } from '../modelConnectionTestPlan'; +import { resolveImageEndpoint, withImageServiceAuth } from './imageDialects'; + +/** Compare request behavior, not the generated record id or display name. */ +export function sameImageService(left: ImageServiceConfig, right: ImageServiceConfig): boolean { + const comparable = (value: ImageServiceConfig) => { + const service = withImageServiceAuth(value, value.auth ?? { type: 'api_key' }); + const params = Object.fromEntries(Object.entries(service.params ?? {}).filter(([, entry]) => entry !== undefined && entry !== '')); + return { + dialect: service.dialect, + endpoint: resolveImageEndpoint(service.base_url, service.request_url, service.dialect).replace(/\/+$/, ''), + model: service.model_name.trim(), + auth: service.auth, + apiKey: service.auth?.type === 'api_key' ? service.api_key.trim() : '', + headers: service.custom_headers ?? {}, + headersMode: service.custom_headers_mode ?? 'merge', + skipSslVerify: service.skip_ssl_verify ?? false, + params: { ...params, n: params.n ?? 1 }, + }; + }; + // Credentials are compared in memory only; never use this value as a DOM id or log it. + return stableJson(comparable(left)) === stableJson(comparable(right)); +} + +export function duplicateImageService(services: readonly ImageServiceConfig[], draft: ImageServiceConfig): ImageServiceConfig | undefined { + const original = services.find(service => service.id === draft.id); + // Existing duplicate records remain editable; only adding or changing identity can introduce a duplicate. + if (original && sameImageService(original, draft)) return undefined; + return services.find(service => service.id !== draft.id && sameImageService(service, draft)); +} + +export class DuplicateImageServiceError extends Error { + constructor() { super('An identical image service is already configured'); } +} + +export function normalizeMedia(value: MediaConfig | null | undefined): MediaConfig { + return { ...value, image_services: value?.image_services ?? [] }; +} + +export function effectiveImageServiceId(media: MediaConfig): string { + const enabled = media.image_services.filter(service => service.enabled); + if (enabled.some(service => service.id === media.default_image_service)) return media.default_image_service!; + return enabled[0]?.id ?? ''; +} + +export function withDefaultImageService(media: MediaConfig): MediaConfig { + const id = effectiveImageServiceId(media); + return id && id !== media.default_image_service ? { ...media, default_image_service: id } : media; +} + +export function imageModelLabel(service: ImageServiceConfig): string { + if (service.name && service.model_name && service.name !== service.model_name) return `${service.name} · ${service.model_name}`; + return service.name || service.model_name || 'ChatGPT'; +} + +/** A configured Codex model drives the subscription image tool directly. */ +export function configuredSubscriptionImageModels(models: readonly AIModelConfig[]): AIModelConfig[] { + return models.filter(model => model.enabled && model.id && model.auth?.type === 'subscription' && model.auth.provider === 'codex'); +} + +export function imageServiceFromSubscriptionModel(model: AIModelConfig): ImageServiceConfig { + if (model.auth?.type !== 'subscription' || model.auth.provider !== 'codex') { + throw new Error('This configured model does not support subscription image generation'); + } + return { + id: `configured-image:${model.id}`, name: model.name || model.model_name, + dialect: 'chatgpt_codex', base_url: 'https://chatgpt.com/backend-api/codex', + model_name: model.model_name, api_key: '', auth: { ...model.auth }, enabled: true, + params: { n: 1 }, skip_ssl_verify: model.skip_ssl_verify, + }; +} + +export function matchesImageModel(service: ImageServiceConfig, query: string, capability: string, provider: string): boolean { + if (capability && capability !== 'image_generation') return false; + if (provider && provider !== `image:${service.dialect}`) return false; + const search = query.trim().toLowerCase(); + return !search || [service.name, service.model_name, service.dialect].join(' ').toLowerCase().includes(search); +} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/useImageGenerationSettings.ts b/src/web-ui/src/infrastructure/config/components/image-generation/useImageGenerationSettings.ts new file mode 100644 index 0000000000..5338291d32 --- /dev/null +++ b/src/web-ui/src/infrastructure/config/components/image-generation/useImageGenerationSettings.ts @@ -0,0 +1,173 @@ +import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react'; +import { configManager } from '../../services/ConfigManager'; +import { useConfigSeed } from '../../hooks/useConfigSeed'; +import { getProviderDisplayName, getProviderInstanceId } from '../../services/modelConfigs'; +import type { AIModelConfig, ImageServiceConfig, MediaConfig } from '../../types'; +import { getActiveSurfaceScope, onSurfaceActivated } from '@/infrastructure/peer-device/deviceSurface'; +import { confirmDialog } from '@/infrastructure/confirm-dialog'; +import { useI18n } from '@/infrastructure/i18n'; +import { useNotification } from '@/shared/notification-system'; +import { createLogger } from '@/shared/utils/logger'; +import { configuredSubscriptionImageModels, duplicateImageService, DuplicateImageServiceError, effectiveImageServiceId, imageServiceFromSubscriptionModel, normalizeMedia, sameImageService, withDefaultImageService } from './imageModels'; +import type { ReusableImageCredential } from './ImageGenerationEditor'; +import { mediaAPI } from '@/infrastructure/api/service-api/MediaApi'; + +const log = createLogger('ImageGenerationSettings'); + +export function useImageGenerationSettings() { + const { t, formatNumber } = useI18n('settings/media'); + const { success, error } = useNotification(); + const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); + const seed = useConfigSeed(['media', 'ai.models']); + const [models, setModels] = useState(() => seed.get('ai.models', [])); + const [media, setMedia] = useState(() => normalizeMedia(seed.get('media', null))); + const [loading, setLoading] = useState(!seed.loaded); + const [loadError, setLoadError] = useState(false); + const [saving, setSaving] = useState(false); + const [editor, setEditor] = useState<{ initial: ImageServiceConfig | null }>(); + const [testingId, setTestingId] = useState(); + const testingRef = useRef(false); + const savingRef = useRef(false); + const request = useRef(0); + const previousEpoch = useRef(scope.epoch); + const loadedEpoch = useRef(seed.loaded ? scope.epoch : undefined); + const defaultAttempt = useRef(''); + + const load = useCallback(async () => { + const ticket = ++request.current; + try { + const [value, configuredModels] = await Promise.all([ + configManager.getConfig('media'), + configManager.getOptionalConfig('ai.models'), + ]); + if (!scope.isCurrent() || ticket !== request.current) return; + loadedEpoch.current = scope.epoch; + setMedia(normalizeMedia(value)); setModels(configuredModels ?? []); setLoadError(false); + } catch (cause) { + if (scope.isCurrent() && ticket === request.current) { + log.error('Failed to load image models', cause); setLoadError(true); + } + } finally { + if (scope.isCurrent() && ticket === request.current) setLoading(false); + } + }, [scope]); + + useEffect(() => { + const changed = previousEpoch.current !== scope.epoch; + previousEpoch.current = scope.epoch; + if (changed) { loadedEpoch.current = undefined; defaultAttempt.current = ''; setMedia(normalizeMedia(null)); setModels([]); setEditor(undefined); setSaving(false); savingRef.current = false; setLoading(true); setTestingId(undefined); testingRef.current = false; } + if (changed || !seed.loaded) void load(); + const unwatchMedia = configManager.watch('media', () => { void load(); }); + const unwatchModels = configManager.watch('ai.models', () => { void load(); }); + return () => { unwatchMedia(); unwatchModels(); }; + }, [load, scope.epoch, seed.loaded]); + + const mutate = useCallback(async (update: (value: MediaConfig) => MediaConfig, notifySaved = true): Promise => { + if (savingRef.current || loading || loadError || !scope.isCurrent()) return false; + savingRef.current = true; setSaving(true); + // Invalidate reads begun before this mutation; watches reload the committed value. + request.current += 1; + try { + const value = await configManager.updateConfig('media', current => { + const normalized = normalizeMedia(current); + return scope.isCurrent() ? withDefaultImageService(update(normalized)) : normalized; + }); + if (!scope.isCurrent()) return false; + setMedia(value); if (notifySaved) success(t('messages.saved')); return true; + } catch (cause) { + if (scope.isCurrent()) { + if (cause instanceof DuplicateImageServiceError) error(t('wizard.validation.duplicate')); + else { log.error('Failed to save image models', cause); error(t('messages.saveFailed')); } + } + return false; + } finally { + if (scope.isCurrent()) { savingRef.current = false; setSaving(false); } + } + }, [loading, loadError, scope, success, error, t]); + + useEffect(() => { + if (loadedEpoch.current !== scope.epoch || loading || loadError || saving || !scope.isCurrent()) return; + const id = effectiveImageServiceId(media); + if (!id || id === media.default_image_service) return; + const attempt = JSON.stringify([scope.epoch, media.default_image_service, media.image_services.filter(service => service.enabled).map(service => service.id)]); + // A failed automatic save stays manually retryable without a render-driven retry loop. + if (defaultAttempt.current === attempt) return; + defaultAttempt.current = attempt; + void mutate(value => value, false); + }, [media, loading, loadError, saving, scope, mutate]); + + const saveService = async (service: ImageServiceConfig): Promise => { + const saved = await mutate(value => { + if (duplicateImageService(value.image_services, service)) throw new DuplicateImageServiceError(); + const exists = value.image_services.some(item => item.id === service.id); + const services = exists ? value.image_services.map(item => item.id === service.id ? service : item) : [...value.image_services, service]; + return { ...value, image_services: services }; + }); + if (saved) setEditor(undefined); + return saved; + }; + const selectDefault = async (id: string) => { + const model = id.startsWith('configured:') ? models.find(item => item.id === id.slice('configured:'.length)) : undefined; + if (id.startsWith('configured:') && (!model || !model.enabled)) return; + await mutate(value => { + if (!model) return { ...value, default_image_service: id || undefined }; + const serviceId = `configured-image:${model.id}`; + const candidate = value.image_services.find(item => item.id === serviceId) ?? imageServiceFromSubscriptionModel(model); + const service = { ...(value.image_services.find(item => sameImageService(item, candidate)) ?? candidate), enabled: true }; + return { ...value, image_services: [...value.image_services.filter(item => item.id !== service.id), service], default_image_service: service.id }; + }); + }; + const toggle = async (service: ImageServiceConfig, enabled: boolean) => { + await mutate(value => ({ ...value, image_services: value.image_services.map(item => item.id === service.id ? { ...item, enabled } : item), + default_image_service: !enabled && value.default_image_service === service.id ? undefined : value.default_image_service })); + }; + const remove = async (service: ImageServiceConfig) => { + const currentScope = scope; + if (!await confirmDialog({ title: t('delete.title'), message: t('delete.message', { name: service.name || service.model_name }), confirmText: t('delete.confirm'), type: 'warning' }) || !currentScope.isCurrent()) return; + await mutate(value => ({ ...value, image_services: value.image_services.filter(item => item.id !== service.id), + default_image_service: value.default_image_service === service.id ? undefined : value.default_image_service })); + }; + const test = async (service: ImageServiceConfig) => { + if (testingRef.current || savingRef.current || !scope.isCurrent()) return; + testingRef.current = true; setTestingId(service.id); + try { + const result = await mediaAPI.testImageGenerationService(service); + if (!scope.isCurrent()) return; + if (result.success) success(t('wizard.test.success', { time: formatNumber(result.elapsed_ms) })); + else error(result.message || t('wizard.errors.provider')); + } catch (cause) { + if (scope.isCurrent()) error(cause instanceof Error ? cause.message : t('wizard.errors.transport')); + } finally { + if (scope.isCurrent()) { testingRef.current = false; setTestingId(undefined); } + } + }; + const credentials = useMemo(() => { + const providers = new Map(); + for (const model of models) { + if (!model.id?.trim() || !model.enabled || model.auth?.type === 'subscription_pool' || (model.auth?.type === 'subscription' && model.auth.provider !== 'codex')) continue; + // Legacy models have no provider instance id: compare their connection facts, + // while keeping credentials out of option values and the DOM. + const key = getProviderInstanceId(model) ?? JSON.stringify([ + model.provider, model.base_url?.trim().replace(/\/+$/, ''), model.auth ?? { type: 'api_key' }, model.api_key ?? '', + Object.entries(model.custom_headers ?? {}).sort(([left], [right]) => left.localeCompare(right)), + model.custom_headers_mode, model.skip_ssl_verify, + ]); + if (!providers.has(key)) providers.set(key, { + id: getProviderInstanceId(model) ?? model.id, label: getProviderDisplayName(model), + apiKey: model.api_key ?? '', baseUrl: model.base_url ?? '', auth: model.auth ?? { type: 'api_key' }, + customHeaders: model.custom_headers, customHeadersMode: model.custom_headers_mode, skipSslVerify: model.skip_ssl_verify, + }); + } + return [...providers.values()]; + }, [models]); + + return { scopeEpoch: scope.epoch, media, loading, loadError, saving, load, editor, credentials, testingId, test, + selectedId: effectiveImageServiceId(media), + configuredModels: configuredSubscriptionImageModels(models).filter(model => !media.image_services.some(service => + service.auth?.type === 'subscription' && service.auth.provider === 'codex' && model.auth?.type === 'subscription' + && service.auth.account_id === model.auth.account_id && service.enabled && service.model_name === model.model_name)), + openCreate: () => setEditor({ initial: null }), openEdit: (initial: ImageServiceConfig) => setEditor({ initial }), + closeEditor: () => { if (!savingRef.current) setEditor(undefined); }, saveService, selectDefault, toggle, remove }; +} + +export type ImageGenerationSettings = ReturnType; diff --git a/src/web-ui/src/infrastructure/config/types/index.ts b/src/web-ui/src/infrastructure/config/types/index.ts index d7bf7a8820..44bdd0d343 100644 --- a/src/web-ui/src/infrastructure/config/types/index.ts +++ b/src/web-ui/src/infrastructure/config/types/index.ts @@ -8,6 +8,8 @@ export interface GlobalConfig { terminal: TerminalConfig; workspace: WorkspaceConfig; ai: AIConfig; + /** Image/media services. Deliberately a sibling of `ai`, never a chapter of it. */ + media: MediaConfig; tool_permissions: ToolPermissionConfig; memories: MemoriesConfig; schema_version: 1; @@ -363,6 +365,61 @@ export interface DefaultModelsConfig { speech_recognition?: string | null; } +/** + * Wire request dialect of an image service. + * + * The dialect decides the endpoint path, body shape and result shape, so it is + * chosen explicitly instead of being inferred from the model name. + */ +/** + * Image request protocol. + * + * `chatgpt_codex` is the subscription route: it drives a chat model's + * `image_generation` tool over the provider's own backend and never touches the + * API-key image endpoint. + */ +export type ImageDialect = 'openai_images' | 'dashscope_native' | 'chatgpt_codex'; + +/** Default generation parameters persisted per service. */ +export interface ImageGenerationParams { + /** Dialect-native size expression (`1024x1024` for OpenAI, `2048*2048` for DashScope). */ + size?: string; + n?: number; + negative_prompt?: string; + quality?: string; + background?: string; + output_format?: string; + watermark?: boolean; + prompt_extend?: boolean; + seed?: number; +} + +/** One independently configured image generation service. */ +export interface ImageServiceConfig { + id: string; + name: string; + dialect: ImageDialect; + /** Service root without the dialect endpoint path. */ + base_url: string; + /** Full request URL override; replaces `base_url` + dialect path when set. */ + request_url?: string; + model_name: string; + api_key: string; + auth?: AuthConfig; + enabled: boolean; + params: ImageGenerationParams; + custom_headers?: Record; + custom_headers_mode?: CustomHeadersMode; + skip_ssl_verify?: boolean; +} + +/** Media service configuration stored beside `ai`, not inside it. */ +export interface MediaConfig { + image_services: ImageServiceConfig[]; + /** Service id used by `generate_image` when a call names no service. */ + default_image_service?: string | null; +} + export type SubagentModelSelection = | { kind: 'fixed'; model_id: string } | { kind: 'inherit' }; diff --git a/src/web-ui/src/infrastructure/i18n/presets/namespaceRegistry.ts b/src/web-ui/src/infrastructure/i18n/presets/namespaceRegistry.ts index 2f1fa74a13..23dab8bbf9 100644 --- a/src/web-ui/src/infrastructure/i18n/presets/namespaceRegistry.ts +++ b/src/web-ui/src/infrastructure/i18n/presets/namespaceRegistry.ts @@ -35,6 +35,7 @@ export const ALL_NAMESPACES = [ 'settings/hooks', 'settings/mcp', 'settings/mcp-tools', + 'settings/media', 'settings/memory', 'settings/models', 'settings/quick-actions', diff --git a/src/web-ui/src/locales/en-US/flow-chat.json b/src/web-ui/src/locales/en-US/flow-chat.json index fcedae3edf..dadc2793b1 100644 --- a/src/web-ui/src/locales/en-US/flow-chat.json +++ b/src/web-ui/src/locales/en-US/flow-chat.json @@ -3025,6 +3025,7 @@ "AgentDelete": "Delete agents", "SessionHistory": "Export history", "analyze_image": "Analyze image", + "generate_image": "Generate image", "GetTime": "Read time", "ListMCPResources": "Discover resources", "ReadMCPResource": "Read resource", @@ -3060,6 +3061,7 @@ "includeToolInputs": "Include tool inputs", "includeThinking": "Include thinking", "model": "Vision model", + "service": "Service", "dimensions": "Image size", "question": "Question", "analysis": "Analysis", @@ -3283,6 +3285,7 @@ "imageCurrent": "Current source image", "imageUnavailable": "Image preview is unavailable. You can open the recorded path to inspect the source.", "imageLoading": "Loading source image…", + "generatedImages": "Images generated: {{value}}", "hostUnavailable": "This record belongs to a dispatched host. Open the resource on that host; local navigation is unavailable.", "remoteLoopback": "This loopback address belongs to the executing host. Open it on that host.", "desktopCard": { diff --git a/src/web-ui/src/locales/en-US/settings/default-model.json b/src/web-ui/src/locales/en-US/settings/default-model.json index da86d6a0a6..a51a151e3f 100644 --- a/src/web-ui/src/locales/en-US/settings/default-model.json +++ b/src/web-ui/src/locales/en-US/settings/default-model.json @@ -32,7 +32,7 @@ "description": "Analyze images and screenshots, support mixed image-text conversations" }, "image_generation": { - "label": "Image Generation", + "label": "Image generation model", "description": "Generate images from text descriptions" }, "speech_recognition": { diff --git a/src/web-ui/src/locales/en-US/settings/media.json b/src/web-ui/src/locales/en-US/settings/media.json new file mode 100644 index 0000000000..de8600fba4 --- /dev/null +++ b/src/web-ui/src/locales/en-US/settings/media.json @@ -0,0 +1,205 @@ +{ + "title": "Image generation", + "subtitle": "Image services are separate from chat models: whenever an image is generated, the service selected here is used, whatever model the session is running.", + "loadFailed": "Unable to load image generation settings.", + "retry": "Retry", + "default": { + "label": "Default service", + "description": "Default model for generating images. You can also choose an existing Codex subscription model.", + "placeholder": "Choose an image model", + "unavailable": "Selected model unavailable" + }, + "empty": { + "title": "No image service configured yet", + "description": "Add a service to let the agent generate images. Until then, image generation stays unavailable." + }, + "card": { + "defaultBadge": "Default", + "model": "Model", + "endpoint": "Endpoint", + "toggleEnabled": "Enable this service", + "apiKeyBadge": "API key", + "subscriptionBadge": "Subscription · {{provider}}" + }, + "actions": { + "add": "Add service", + "delete": "Delete", + "cancel": "Cancel", + "save": "Save" + }, + "messages": { + "saved": "Image service settings saved.", + "saveFailed": "Unable to save image service settings.", + "deleted": "Image service deleted." + }, + "delete": { + "title": "Delete image service", + "message": "Delete \"{{name}}\"? Images already generated stay on disk.", + "confirm": "Delete" + }, + "dialect": { + "openaiImages": { + "name": "OpenAI Images", + "description": "The `POST /images/generations` JSON API. Works with OpenAI and with most compatible relay services." + }, + "dashscopeNative": { + "name": "Qwen native (DashScope)", + "description": "Alibaba Cloud DashScope multimodal generation, used by Qwen image models." + }, + "chatgptCodex": { + "name": "ChatGPT subscription (Codex)", + "description": "Generates through a ChatGPT Plus / Pro / Team subscription: a chat model calls its built-in image tool and answers with a stream of events. This is a separate billing path from an API key." + } + }, + "subscriptionLabels": { + "codex": "Codex", + "antigravity": "Antigravity", + "opencode": "OpenCode", + "grok": "Grok", + "hermes": "Hermes" + }, + "wizard": { + "titleCreate": "Add or import an image model", + "titleEdit": "Edit image model", + "guideLabel": "Setup steps", + "progress": "{{current}} / {{total}}", + "back": "Back", + "next": "Continue", + "steps": { + "dialect": { + "label": "Protocol", + "title": "Which protocol does the service speak?", + "description": "The protocol decides the request body, the endpoint path and the result shape. Pick the one the provider documents for image generation." + }, + "connection": { + "label": "Connection", + "title": "Where is it and how do we authenticate?", + "description": "Enter the service root, the credential and the image model id. A relay service needs its own base URL." + }, + "parameters": { + "label": "Parameters", + "title": "What should every image request default to?", + "description": "These values apply whenever a request does not override them. Size syntax follows the selected protocol." + }, + "test": { + "label": "Check", + "title": "Generate one image to confirm the setup", + "description": "This is the only reliable check: it proves the endpoint, credential, model and protocol all match." + } + }, + "auth": { + "label": "Authentication", + "apiKey": "API key", + "subscription": "Subscription login", + "apiKeyHint": "Use this service's own key, stored locally with the rest of your settings.", + "subscriptionHint": "Uses the imported {{provider}} login and resolves its access token while sending.", + "subscriptionCaveat": "A subscription login only covers what the provider opens to it, usually its own chat endpoints. For example, OpenAI's image endpoint requires the api.model.images.request scope, which chat subscription logins normally lack. The check returns the provider's own error in that case; use an API key instead.", + "subscriptionRouteHint": "This protocol is driven by the {{provider}} subscription login; the token is resolved while sending.", + "subscriptionRouteNote": "This path spends subscription allowance, not API key credit. The address, the model catalog and the access token all come from the login, so nothing needs to be filled in here. If the login has expired, re-authorize it under subscription logins first." + }, + "reuse": { + "label": "Reuse a configured credential", + "hint": "Pick a chat model to copy its key and address, or pick a subscription model to reuse that login. Review them before saving.", + "none": "Do not reuse" + }, + "fields": { + "endpoint": { + "label": "Effective request address", + "unresolved": "Shown once a service address is entered" + }, + "baseUrl": { + "label": "Service address", + "hint": "The request is sent to this address plus {{path}}.", + "subscriptionHint": "Leave it empty to use the address the subscription login returns. If that address does not serve images, enter the image service address explicitly.", + "subscriptionRouteHint": "Supplied by the subscription login and not editable. After saving, the address actually used is whatever the login returns." + }, + "requestUrl": { + "label": "Full request address (optional)", + "hint": "Replaces service address plus endpoint path. Use it for a dedicated instance or a gateway." + }, + "apiKey": { + "label": "API key", + "hint": "Stored locally with the rest of your settings." + }, + "modelName": { + "label": "Image model id", + "hint": "The exact model id the provider expects for image generation, not a chat model.", + "subscriptionPlaceholder": "Empty uses any model the account offers", + "subscriptionHint": "This is the chat model that drives the image tool, not an image model name. Leave it empty to pick one from the account's catalog automatically." + }, + "name": { + "label": "Display name", + "hint": "Optional. Defaults to the model id." + }, + "size": { + "label": "Default size", + "hint": "Sizes are written in {{dialect}} syntax." + }, + "n": { + "label": "Images per request", + "hint": "Between 1 and {{max}}." + }, + "quality": { + "label": "Quality" + }, + "background": { + "label": "Background" + }, + "negativePrompt": { + "label": "Negative prompt", + "hint": "Content the model should avoid." + }, + "promptExtend": { + "label": "Rewrite the prompt", + "hint": "Lets the provider expand short prompts into a fuller description." + }, + "watermark": { + "label": "Add a watermark" + } + }, + "validation": { + "unsupportedAuth": "Image generation does not support subscription account pools. Choose a single account.", + "duplicate": "An identical image configuration already exists. Edit the existing entry instead.", + "baseUrlInvalid": "Enter a valid http or https address.", + "modelRequired": "Enter the image model id.", + "apiKeyRequired": "Enter the API key." + }, + "test": { + "billed": "The check generates a real image and is billed by your provider.", + "promptLabel": "Prompt used for the check", + "run": "Generate test image", + "running": "Generating...", + "success": "Image generated in {{time}} ms.", + "failed": "The check failed.", + "previewAlt": "Image generated by the test" + }, + "errors": { + "config": "This service is incomplete. Check the address, credential and model id.", + "auth": "The provider rejected the credential. Check that it belongs to this address, and whether the subscription login may call the image endpoint.", + "invalid_request": "The provider rejected the request. This usually means the protocol does not match the service, or a parameter is not supported.", + "content_filtered": "The provider's safety system blocked this prompt. Try a different subject.", + "rate_limited": "The provider is rate limiting or the quota is exhausted. Try again later.", + "timeout": "The provider did not answer in time.", + "transport": "The address could not be reached. If the service is only reachable through a local proxy, leave the proxy empty to follow the system proxy, or set the proxy address in the proxy settings.", + "missingScope": "The credential authenticated but lacks the image permission (the provider requires a scope such as api.model.images.request). A subscription login usually covers only its own chat endpoints, so use the provider API key for image generation.", + "provider": "The provider reported an internal error.", + "empty_result": "The provider answered but returned no image." + } + }, + "acquisition": { + "label": "Image models", + "description": "Add an image service or import an existing model configuration", + "add": "Add or import", + "empty": "No image models added", + "remove": "Delete image model {{name}}" + }, + "import": { + "source": "Configuration source", + "manual": "Add a service manually", + "saved": "Current configuration", + "hint": "Reuse an existing provider connection or configure an image service.", + "subscriptionHint": "Uses your imported subscription account. No additional key or address is needed." + }, + "advanced": "Advanced settings", + "automatic": "Model default" +} diff --git a/src/web-ui/src/locales/en-US/settings/models.json b/src/web-ui/src/locales/en-US/settings/models.json index c989c705c2..8d85ea012e 100644 --- a/src/web-ui/src/locales/en-US/settings/models.json +++ b/src/web-ui/src/locales/en-US/settings/models.json @@ -499,7 +499,7 @@ }, "proxy": { "enable": "Enable Proxy", - "enableHint": "When enabled, all AI requests are sent through the proxy (requires a OpenBitFun restart)", + "enableHint": "With the address left empty, requests follow the operating system proxy, such as the system proxy a local proxy agent sets. With an address and the switch on, every AI request uses that proxy; an address with the switch off forces a direct connection.", "url": "Proxy URL", "urlPlaceholder": "http://127.0.0.1:7890 or socks5://127.0.0.1:1080", "urlHint": "Supports HTTP/HTTPS/SOCKS5 proxy, format: protocol://address:port", diff --git a/src/web-ui/src/locales/zh-CN/flow-chat.json b/src/web-ui/src/locales/zh-CN/flow-chat.json index 5502dccca7..0755944cf3 100644 --- a/src/web-ui/src/locales/zh-CN/flow-chat.json +++ b/src/web-ui/src/locales/zh-CN/flow-chat.json @@ -3025,6 +3025,7 @@ "AgentDelete": "删除子代理", "SessionHistory": "导出会话历史", "analyze_image": "分析图片", + "generate_image": "生成图片", "GetTime": "读取时间", "ListMCPResources": "发现资源", "ReadMCPResource": "读取资源", @@ -3060,6 +3061,7 @@ "includeToolInputs": "包含工具参数", "includeThinking": "包含思考", "model": "视觉模型", + "service": "服务", "dimensions": "图片尺寸", "question": "分析问题", "analysis": "分析结果", @@ -3283,6 +3285,7 @@ "imageCurrent": "当前源图像", "imageUnavailable": "无法预览图像，可打开记录的路径查看源文件。", "imageLoading": "正在读取源图像…", + "generatedImages": "已生成图片：{{value}} 张", "hostUnavailable": "这条记录来自被派发任务的主机，请在该主机打开资源；当前无法从本机跳转。", "remoteLoopback": "这个回环地址属于执行主机，请在该主机上打开。", "desktopCard": { diff --git a/src/web-ui/src/locales/zh-CN/settings/default-model.json b/src/web-ui/src/locales/zh-CN/settings/default-model.json index 4a2f9290e2..fa45f8c701 100644 --- a/src/web-ui/src/locales/zh-CN/settings/default-model.json +++ b/src/web-ui/src/locales/zh-CN/settings/default-model.json @@ -32,7 +32,7 @@ "description": "当主模型不支持图片输入时，用于分析图片和截图内容" }, "image_generation": { - "label": "图像生成", + "label": "图像生成模型", "description": "根据文字描述生成图片" }, "speech_recognition": { diff --git a/src/web-ui/src/locales/zh-CN/settings/media.json b/src/web-ui/src/locales/zh-CN/settings/media.json new file mode 100644 index 0000000000..18b14d39b4 --- /dev/null +++ b/src/web-ui/src/locales/zh-CN/settings/media.json @@ -0,0 +1,205 @@ +{ + "title": "图像生成", + "subtitle": "图像服务与对话模型相互独立：无论会话正在使用哪个模型，生成图片时都会使用这里选定的服务。", + "loadFailed": "无法加载图像生成设置。", + "retry": "重试", + "default": { + "label": "默认服务", + "description": "生成图片时使用的默认模型，也可直接选择已配置的 Codex 订阅模型。", + "placeholder": "选择图像生成模型", + "unavailable": "已选择的模型不可用" + }, + "empty": { + "title": "尚未配置图像服务", + "description": "添加一个服务后，智能体才能生成图片；在此之前图像生成保持不可用。" + }, + "card": { + "defaultBadge": "默认", + "model": "模型", + "endpoint": "接口地址", + "toggleEnabled": "启用此服务", + "apiKeyBadge": "API 密钥", + "subscriptionBadge": "订阅 · {{provider}}" + }, + "actions": { + "add": "添加服务", + "delete": "删除", + "cancel": "取消", + "save": "保存" + }, + "messages": { + "saved": "图像服务设置已保存。", + "saveFailed": "无法保存图像服务设置。", + "deleted": "图像服务已删除。" + }, + "delete": { + "title": "删除图像服务", + "message": "确定删除「{{name}}」吗？已生成的图片仍保留在磁盘上。", + "confirm": "删除" + }, + "dialect": { + "openaiImages": { + "name": "OpenAI Images", + "description": "即 `POST /images/generations` 这个 JSON 接口。适用于 OpenAI 以及大多数兼容中转服务。" + }, + "dashscopeNative": { + "name": "通义千问原生（DashScope）", + "description": "阿里云百炼 DashScope 的多模态生成接口，供千问图像模型使用。" + }, + "chatgptCodex": { + "name": "ChatGPT 订阅（Codex）", + "description": "使用 ChatGPT Plus / Pro / Team 订阅出图：请求由对话模型调用内置图像工具完成，响应是流式事件。与 API 密钥计费是两条不同的路。" + } + }, + "subscriptionLabels": { + "codex": "Codex", + "antigravity": "Antigravity", + "opencode": "OpenCode", + "grok": "Grok", + "hermes": "Hermes" + }, + "wizard": { + "titleCreate": "添加或导入图像模型", + "titleEdit": "编辑图像模型", + "guideLabel": "配置步骤", + "progress": "{{current}} / {{total}}", + "back": "上一步", + "next": "继续", + "steps": { + "dialect": { + "label": "协议", + "title": "这个服务使用哪种协议？", + "description": "协议决定了请求体、接口路径和返回结构。请选择服务商文档中标注用于图像生成的那一种。" + }, + "connection": { + "label": "连接", + "title": "地址是什么，如何鉴权？", + "description": "填写服务根地址、凭据和图像模型 ID。中转服务必须填它自己的地址。" + }, + "parameters": { + "label": "参数", + "title": "每次生成默认使用什么参数？", + "description": "这里的值作为默认值生效，单次请求可以覆盖。尺寸写法跟随所选的协议。" + }, + "test": { + "label": "验证", + "title": "生成一张图片来确认配置", + "description": "这是唯一可靠的验证方式：它能同时证明地址、凭据、模型和协议是匹配的。" + } + }, + "auth": { + "label": "鉴权方式", + "apiKey": "API 密钥", + "subscription": "订阅登录", + "apiKeyHint": "使用这个服务自己的密钥，与其他设置一起保存在本地。", + "subscriptionHint": "使用已导入的 {{provider}} 登录，请求时自动解析访问令牌。", + "subscriptionCaveat": "订阅登录由服务商按其产品开放范围授权，一般只覆盖自家对话接口。例如 OpenAI 图像接口要求 api.model.images.request 权限，对话类订阅登录通常不含该权限。测试会原样返回服务商错误，此时请改用 API 密钥。", + "subscriptionRouteHint": "该协议由 {{provider}} 订阅登录驱动，令牌在发送时解析。", + "subscriptionRouteNote": "这条路径消耗的是订阅额度，不是 API 密钥额度。地址、模型目录和访问令牌都由订阅登录提供，无需手工填写；若登录失效，请先在订阅登录处重新授权。" + }, + "reuse": { + "label": "复用已配置的凭据", + "hint": "选中一个对话模型即可复制它的密钥与地址；选中订阅模型则复用该订阅登录。保存前请确认是否正确。", + "none": "不复用" + }, + "fields": { + "endpoint": { + "label": "实际请求地址", + "unresolved": "填写服务地址后显示" + }, + "baseUrl": { + "label": "服务地址", + "hint": "请求会发往该地址加上 {{path}}。", + "subscriptionHint": "留空则使用订阅登录返回的地址；若该地址不提供图像接口，请显式填写图像服务地址。", + "subscriptionRouteHint": "由订阅登录解析提供，不可编辑。保存后实际发往的地址以订阅返回值为准。" + }, + "requestUrl": { + "label": "完整请求地址（可选）", + "hint": "填写后替代「服务地址 + 接口路径」，用于专有实例或网关。" + }, + "apiKey": { + "label": "API 密钥", + "hint": "与其他设置一起保存在本地。" + }, + "modelName": { + "label": "图像模型 ID", + "hint": "填服务商用于图像生成的确切模型 ID，不是对话模型。", + "subscriptionPlaceholder": "留空则使用账号可用的模型", + "subscriptionHint": "这里填的是驱动图像工具的对话模型，不是图像模型名；留空会按账号可用模型自动选择。" + }, + "name": { + "label": "显示名称", + "hint": "可留空，默认使用模型 ID。" + }, + "size": { + "label": "默认尺寸", + "hint": "尺寸按 {{dialect}} 的写法填写。" + }, + "n": { + "label": "每次生成张数", + "hint": "取值 1 至 {{max}}。" + }, + "quality": { + "label": "质量" + }, + "background": { + "label": "背景" + }, + "negativePrompt": { + "label": "反向提示词", + "hint": "希望模型避免出现的内容。" + }, + "promptExtend": { + "label": "改写提示词", + "hint": "允许服务商把简短提示扩写为更完整的描述。" + }, + "watermark": { + "label": "添加水印" + } + }, + "validation": { + "unsupportedAuth": "图像生成暂不支持订阅账号池，请选择单个账号。", + "duplicate": "已存在相同的图像生成配置，请编辑已有配置。", + "baseUrlInvalid": "请填写有效的 http 或 https 地址。", + "modelRequired": "请填写图像模型 ID。", + "apiKeyRequired": "请填写 API 密钥。" + }, + "test": { + "billed": "测试会真实生成一张图片，服务商会正常计费。", + "promptLabel": "测试使用的提示词", + "run": "生成测试图片", + "running": "生成中…", + "success": "已生成图片，耗时 {{time}} 毫秒。", + "failed": "测试失败。", + "previewAlt": "测试生成的图片" + }, + "errors": { + "config": "该服务配置不完整，请检查地址、凭据和模型 ID。", + "auth": "服务商拒绝了凭据，请确认该凭据属于这个地址，或该订阅登录能否调用图像接口。", + "invalid_request": "服务商拒绝了请求。通常是协议选错，或某个参数不被支持。", + "content_filtered": "提示词被服务商的安全审核拦截，请更换描述内容。", + "rate_limited": "触发限流或额度已用尽，请稍后再试。", + "timeout": "服务商未在预期时间内响应。", + "transport": "无法连接到该地址。若必须经本地代理才能访问该服务，请保持代理为空以跟随系统代理，或在代理设置中填写代理地址。", + "missingScope": "该凭据已通过身份验证，但缺少图像接口权限（服务商要求 api.model.images.request 等范围）。订阅登录通常只覆盖自家对话接口，请改用服务商的 API 密钥。", + "provider": "服务商返回内部错误。", + "empty_result": "服务商响应成功但没有返回图片。" + } + }, + "acquisition": { + "label": "图像生成模型", + "description": "添加图像服务，或导入已有模型配置", + "add": "添加或导入", + "empty": "尚未添加图像模型", + "remove": "删除图像模型 {{name}}" + }, + "import": { + "source": "配置来源", + "manual": "手动添加服务", + "saved": "当前配置", + "hint": "复用已有服务提供方的连接，或手动配置图像服务。", + "subscriptionHint": "使用已导入的订阅账号，无需重复填写密钥和地址。" + }, + "advanced": "高级设置", + "automatic": "跟随模型默认值" +} diff --git a/src/web-ui/src/locales/zh-CN/settings/models.json b/src/web-ui/src/locales/zh-CN/settings/models.json index 25bab91b0b..9df92914ff 100644 --- a/src/web-ui/src/locales/zh-CN/settings/models.json +++ b/src/web-ui/src/locales/zh-CN/settings/models.json @@ -499,7 +499,7 @@ }, "proxy": { "enable": "启用代理", - "enableHint": "启用后所有 AI 请求将通过代理发送（需重启 OpenBitFun 生效）", + "enableHint": "留空代理地址时，请求会跟随操作系统代理（例如本地代理软件设置的系统代理）。填写地址并启用后，所有 AI 请求都通过该代理发送；填写地址但关闭开关表示强制直连。", "url": "代理地址", "urlPlaceholder": "http://127.0.0.1:7890 或 socks5://127.0.0.1:1080", "urlHint": "支持 HTTP/HTTPS/SOCKS5 代理，格式：协议://地址:端口", diff --git a/src/web-ui/src/locales/zh-TW/flow-chat.json b/src/web-ui/src/locales/zh-TW/flow-chat.json index 254d857384..031782a125 100644 --- a/src/web-ui/src/locales/zh-TW/flow-chat.json +++ b/src/web-ui/src/locales/zh-TW/flow-chat.json @@ -3025,6 +3025,7 @@ "AgentDelete": "刪除子代理", "SessionHistory": "匯出工作階段歷史", "analyze_image": "分析圖片", + "generate_image": "生成圖片", "GetTime": "讀取時間", "ListMCPResources": "探索資源", "ReadMCPResource": "讀取資源", @@ -3060,6 +3061,7 @@ "includeToolInputs": "包含工具參數", "includeThinking": "包含思考", "model": "視覺模型", + "service": "服務", "dimensions": "圖片尺寸", "question": "分析問題", "analysis": "分析結果", @@ -3283,6 +3285,7 @@ "imageCurrent": "目前來源影像", "imageUnavailable": "無法預覽影像，可開啟記錄的路徑查看來源檔案。", "imageLoading": "正在讀取來源影像…", + "generatedImages": "已生成圖片：{{value}} 張", "hostUnavailable": "這筆記錄來自被派發工作的主機，請在該主機開啟資源；目前無法從本機跳轉。", "remoteLoopback": "這個回環位址屬於執行主機，請在該主機上開啟。", "desktopCard": { diff --git a/src/web-ui/src/locales/zh-TW/settings/default-model.json b/src/web-ui/src/locales/zh-TW/settings/default-model.json index 7bbedf5409..40a05378b0 100644 --- a/src/web-ui/src/locales/zh-TW/settings/default-model.json +++ b/src/web-ui/src/locales/zh-TW/settings/default-model.json @@ -32,7 +32,7 @@ "description": "當主模型不支持圖片輸入時，用於分析圖片和截圖內容" }, "image_generation": { - "label": "圖像生成", + "label": "圖像生成模型", "description": "根據文字描述生成圖片" }, "speech_recognition": { diff --git a/src/web-ui/src/locales/zh-TW/settings/media.json b/src/web-ui/src/locales/zh-TW/settings/media.json new file mode 100644 index 0000000000..3f7e14bc8c --- /dev/null +++ b/src/web-ui/src/locales/zh-TW/settings/media.json @@ -0,0 +1,205 @@ +{ + "title": "圖像生成", + "subtitle": "圖像服務與對話模型彼此獨立：無論工作階段正在使用哪個模型，生成圖片時都會使用這裡選定的服務。", + "loadFailed": "無法載入圖像生成設定。", + "retry": "重試", + "default": { + "label": "預設服務", + "description": "產生圖片時使用的預設模型，也可直接選擇已設定的 Codex 訂閱模型。", + "placeholder": "選擇圖像生成模型", + "unavailable": "已選擇的模型無法使用" + }, + "empty": { + "title": "尚未設定圖像服務", + "description": "新增一個服務後，代理才能生成圖片；在此之前圖像生成維持不可用。" + }, + "card": { + "defaultBadge": "預設", + "model": "模型", + "endpoint": "介面位址", + "toggleEnabled": "啟用此服務", + "apiKeyBadge": "API 金鑰", + "subscriptionBadge": "訂閱 · {{provider}}" + }, + "actions": { + "add": "新增服務", + "delete": "刪除", + "cancel": "取消", + "save": "儲存" + }, + "messages": { + "saved": "圖像服務設定已儲存。", + "saveFailed": "無法儲存圖像服務設定。", + "deleted": "圖像服務已刪除。" + }, + "delete": { + "title": "刪除圖像服務", + "message": "確定刪除「{{name}}」嗎？已生成的圖片仍保留在磁碟上。", + "confirm": "刪除" + }, + "dialect": { + "openaiImages": { + "name": "OpenAI Images", + "description": "即 `POST /images/generations` 這個 JSON 介面。適用於 OpenAI 以及大多數相容中轉服務。" + }, + "dashscopeNative": { + "name": "通義千問原生（DashScope）", + "description": "阿里雲百鍊 DashScope 的多模態生成介面，供千問圖像模型使用。" + }, + "chatgptCodex": { + "name": "ChatGPT 訂閱（Codex）", + "description": "使用 ChatGPT Plus / Pro / Team 訂閱出圖：請求由對話模型呼叫內建圖像工具完成，回應是串流事件。與 API 金鑰計費是兩條不同的路。" + } + }, + "subscriptionLabels": { + "codex": "Codex", + "antigravity": "Antigravity", + "opencode": "OpenCode", + "grok": "Grok", + "hermes": "Hermes" + }, + "wizard": { + "titleCreate": "新增或匯入圖像模型", + "titleEdit": "編輯圖像模型", + "guideLabel": "設定步驟", + "progress": "{{current}} / {{total}}", + "back": "上一步", + "next": "繼續", + "steps": { + "dialect": { + "label": "協定", + "title": "這個服務使用哪種協定？", + "description": "協定決定了請求內容、介面路徑和回應結構。請選擇服務商文件中標註用於圖像生成的那一種。" + }, + "connection": { + "label": "連線", + "title": "位址是什麼，如何驗證？", + "description": "填寫服務根位址、憑證和圖像模型 ID。中轉服務必須填它自己的位址。" + }, + "parameters": { + "label": "參數", + "title": "每次生成預設使用什麼參數？", + "description": "這裡的值作為預設值生效，單次請求可以覆寫。尺寸寫法跟隨所選的協定。" + }, + "test": { + "label": "驗證", + "title": "生成一張圖片來確認設定", + "description": "這是唯一可靠的驗證方式：它能同時證明位址、憑證、模型和協定是相符的。" + } + }, + "auth": { + "label": "驗證方式", + "apiKey": "API 金鑰", + "subscription": "訂閱登入", + "apiKeyHint": "使用這個服務自己的金鑰，與其他設定一起儲存在本機。", + "subscriptionHint": "使用已匯入的 {{provider}} 登入，請求時自動解析存取權杖。", + "subscriptionCaveat": "訂閱登入由服務商按其產品開放範圍授權，一般只涵蓋自家對話介面。例如 OpenAI 圖像介面要求 api.model.images.request 權限，對話類訂閱登入通常不含該權限。測試會原樣回傳服務商錯誤，此時請改用 API 金鑰。", + "subscriptionRouteHint": "此協定由 {{provider}} 訂閱登入驅動，權杖在請求時解析。", + "subscriptionRouteNote": "這條路徑消耗的是訂閱額度，不是 API 金鑰額度。位址、模型目錄與存取權杖都由訂閱登入提供，無需手動填寫；若登入失效，請先在訂閱登入處重新授權。" + }, + "reuse": { + "label": "沿用已設定的憑證", + "hint": "選取一個對話模型即可複製它的金鑰與位址；選取訂閱模型則沿用該訂閱登入。儲存前請確認是否正確。", + "none": "不沿用" + }, + "fields": { + "endpoint": { + "label": "實際請求位址", + "unresolved": "填寫服務位址後顯示" + }, + "baseUrl": { + "label": "服務位址", + "hint": "請求會送往該位址加上 {{path}}。", + "subscriptionHint": "留空則使用訂閱登入回傳的位址；若該位址不提供圖像介面，請明確填寫圖像服務位址。", + "subscriptionRouteHint": "由訂閱登入解析提供，無法編輯。儲存後實際送往的位址以訂閱回傳值為準。" + }, + "requestUrl": { + "label": "完整請求位址（選填）", + "hint": "填寫後取代「服務位址 + 介面路徑」，用於專屬執行個體或閘道。" + }, + "apiKey": { + "label": "API 金鑰", + "hint": "與其他設定一起儲存在本機。" + }, + "modelName": { + "label": "圖像模型 ID", + "hint": "填服務商用於圖像生成的正確模型 ID，不是對話模型。", + "subscriptionPlaceholder": "留空則使用帳號可用的模型", + "subscriptionHint": "這裡填的是驅動圖像工具的對話模型，不是圖像模型名稱；留空會依帳號可用模型自動選擇。" + }, + "name": { + "label": "顯示名稱", + "hint": "可留空，預設使用模型 ID。" + }, + "size": { + "label": "預設尺寸", + "hint": "尺寸按 {{dialect}} 的寫法填寫。" + }, + "n": { + "label": "每次生成張數", + "hint": "取值 1 至 {{max}}。" + }, + "quality": { + "label": "品質" + }, + "background": { + "label": "背景" + }, + "negativePrompt": { + "label": "反向提示詞", + "hint": "希望模型避免出現的內容。" + }, + "promptExtend": { + "label": "改寫提示詞", + "hint": "允許服務商把簡短提示擴寫為更完整的描述。" + }, + "watermark": { + "label": "加入浮水印" + } + }, + "validation": { + "unsupportedAuth": "圖像生成暫不支援訂閱帳號池，請選擇單一帳號。", + "duplicate": "已存在相同的圖像生成設定，請編輯現有設定。", + "baseUrlInvalid": "請填寫有效的 http 或 https 位址。", + "modelRequired": "請填寫圖像模型 ID。", + "apiKeyRequired": "請填寫 API 金鑰。" + }, + "test": { + "billed": "測試會真實生成一張圖片，服務商會正常計費。", + "promptLabel": "測試使用的提示詞", + "run": "生成測試圖片", + "running": "生成中…", + "success": "已生成圖片，耗時 {{time}} 毫秒。", + "failed": "測試失敗。", + "previewAlt": "測試生成的圖片" + }, + "errors": { + "config": "該服務設定不完整，請檢查位址、憑證和模型 ID。", + "auth": "服務商拒絕了憑證，請確認該憑證屬於這個位址，或該訂閱登入能否呼叫圖像介面。", + "invalid_request": "服務商拒絕了請求。通常是協定選錯，或某個參數不被支援。", + "content_filtered": "提示詞被服務商的安全審核攔截，請更換描述內容。", + "rate_limited": "觸發限流或額度已用盡，請稍後再試。", + "timeout": "服務商未在預期時間內回應。", + "transport": "無法連線到該位址。若必須經本機代理才能存取該服務，請保持代理為空以跟隨系統代理，或在代理設定中填寫代理位址。", + "missingScope": "該憑證已通過身分驗證，但缺少圖像介面權限（服務商要求 api.model.images.request 等範圍）。訂閱登入通常只涵蓋自家對話介面，請改用服務商的 API 金鑰。", + "provider": "服務商回傳內部錯誤。", + "empty_result": "服務商回應成功但沒有回傳圖片。" + } + }, + "acquisition": { + "label": "圖像生成模型", + "description": "新增圖像服務，或匯入現有模型設定", + "add": "新增或匯入", + "empty": "尚未新增圖像模型", + "remove": "刪除圖像模型 {{name}}" + }, + "import": { + "source": "設定來源", + "manual": "手動新增服務", + "saved": "目前設定", + "hint": "重用現有服務供應商的連線，或手動設定圖像服務。", + "subscriptionHint": "使用已匯入的訂閱帳號，無需重複填寫金鑰與位址。" + }, + "advanced": "進階設定", + "automatic": "跟隨模型預設值" +} diff --git a/src/web-ui/src/locales/zh-TW/settings/models.json b/src/web-ui/src/locales/zh-TW/settings/models.json index 9d100bf917..903a9e5678 100644 --- a/src/web-ui/src/locales/zh-TW/settings/models.json +++ b/src/web-ui/src/locales/zh-TW/settings/models.json @@ -499,7 +499,7 @@ }, "proxy": { "enable": "啟用代理", - "enableHint": "啟用後所有 AI 請求將透過代理傳送（需重新啟動 OpenBitFun 生效）", + "enableHint": "留空代理位址時，請求會跟隨作業系統代理（例如本機代理軟體設定的系統代理）。填寫位址並啟用後，所有 AI 請求都會透過該代理傳送；填寫位址但關閉開關表示強制直連。", "url": "代理地址", "urlPlaceholder": "http://127.0.0.1:7890 或 socks5://127.0.0.1:1080", "urlHint": "支持 HTTP/HTTPS/SOCKS5 代理，格式：協議://地址:端口", From f0742e3d01a6adce5eb072034ecc140a4cc095be Mon Sep 17 00:00:00 2001 From: kev1n77 Date: Mon, 26 Oct 2026 16:02:34 +0800 Subject: [PATCH 2/3] feat(skills): add built-in image generation workflow --- .../builtin_skills/generate-image/SKILL.md | 49 ++++++++++ .../core/src/agentic/skill_agent_snapshot.rs | 20 +++- .../tools/implementations/skills/catalog.rs | 1 + .../agent-runtime/src/skills/catalog.rs | 4 + .../execution/agent-runtime/src/skills/mod.rs | 8 +- .../agent-runtime/src/skills/policy.rs | 26 ++++- .../agent-runtime/src/skills/selection.rs | 97 +++++++++++++++++++ 7 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 src/crates/assembly/core/builtin_skills/generate-image/SKILL.md diff --git a/src/crates/assembly/core/builtin_skills/generate-image/SKILL.md b/src/crates/assembly/core/builtin_skills/generate-image/SKILL.md new file mode 100644 index 0000000000..8550641f99 --- /dev/null +++ b/src/crates/assembly/core/builtin_skills/generate-image/SKILL.md @@ -0,0 +1,49 @@ +--- +name: generate-image +description: Generate photos, illustrations, posters, and project assets through OpenBitFun's configured default image model. Shape prompts, inspect results, and make targeted refinements using generate_image. Not for image analysis, prompt-only discussions, or edits that must preserve an existing image. +--- + +# Generate Image + +## Define the asset + +Infer purpose, subject, framing, quantity, required text, and destination from context. Ask only for information that blocks success. Preserve detailed requests; enrich brief ones with useful composition, lighting, and material cues without inventing brands, slogans, characters, or story elements. + +Prefer native formats for existing SVGs, icon systems, and editable charts. Deliver actual raster images when the user asks for photos or illustrations. + +## Shape the prompt + +For complex requests, use the following structure, keeping only useful fields. Choose the prompt language for the task; preserve required in-image text verbatim, including its original language and punctuation. + +```text +Purpose: placement, audience, and intended effect +Content: subject, action, setting, and essential details +Style: photo/illustration/3D, materials, lighting, and palette +Composition: viewpoint, framing, crop-safe margins, and necessary negative space +Text: exact copy, hierarchy, placement, and number of occurrences +Constraints: required invariants and important exclusions +``` + +| Asset | Prompt emphasis | +| --- | --- | +| Photography and products | Describe texture, reflections, perspective, contact shadows, and product structure instead of stacking vague quality adjectives. | +| Web assets and covers | Match the actual container and crop; place negative space around existing copy. Keep headings and buttons in page code when appropriate. | +| Posters and infographics | Supply exact copy, verified data, relationships, hierarchy, and reading order. Do not let the model invent facts. | +| Illustrations and sets | Repeat concrete medium, palette, lighting, and subject descriptions. Give each image one clear purpose. | + +Use `view_image` or `analyze_image` to describe visual references when available. The generation tool accepts no reference images, masks, or editing inputs; do not promise preserved identity, exact product details, or unchanged pixels. + +## Generate and inspect + +- Generate only with `generate_image`. Omit `service_id` by default to use the configured default image model. Do not mix in external `imagegen`, `image_gen`, genmedia, or direct API calls. +- Use `n: 1` unless another quantity was requested. Call separately for distinct content; `n` is for variants of one prompt. Track actual returned counts without silently adding candidates. +- Use `size` and `negative_prompt` only when supported by the service. Put style and layout in `prompt`; do not invent tool arguments. +- When inspection is available, check subject and count, structure and cropping, verbatim text, and consistency across sets. Verify actual dimensions, format, and alpha when required; a white or checkerboard background is not transparency. Disclose unverified requirements. +- Refine one specific defect at a time, retaining the original goal, valid constraints, and previous versions for comparison. Regeneration is not a local edit. Stop when the goal is met and stay within the user's quantity and budget. +- Report configuration, authentication, quota, or transport failures without automatically switching services or retrying without cause. For configuration issues, point to Settings → Models → Image generation model. + +## Deliver + +Show or deliver images using returned file paths. Place project assets in the agreed asset directory, update references within the task's scope, and check their actual display. Keep previous versions unless replacement was requested. Operate on remote files through their workspace, not as controller-local files. + +Report delivered quantity, locations, and known limitations, including unmet text, size, transparency, or consistency requirements. diff --git a/src/crates/assembly/core/src/agentic/skill_agent_snapshot.rs b/src/crates/assembly/core/src/agentic/skill_agent_snapshot.rs index 3431c32897..eecafb921d 100644 --- a/src/crates/assembly/core/src/agentic/skill_agent_snapshot.rs +++ b/src/crates/assembly/core/src/agentic/skill_agent_snapshot.rs @@ -107,7 +107,13 @@ async fn build_skill_agent_snapshot( let mut snapshot = TurnSkillAgentSnapshot::default(); if has_tool("Skill") { - snapshot.skills = load_skill_entries(workspace, workspace_services, Some(agent_type)).await; + snapshot.skills = load_skill_entries( + workspace, + workspace_services, + Some(agent_type), + has_tool("generate_image"), + ) + .await; } if has_tool("Task") || has_tool("AgentSpawn") { @@ -183,6 +189,7 @@ async fn load_skill_entries( workspace: Option<&WorkspaceBinding>, workspace_services: Option<&WorkspaceServices>, agent_type: Option<&str>, + image_generation_available: bool, ) -> Vec { let registry = get_skill_registry(); let skills = match workspace { @@ -211,10 +218,13 @@ async fn load_skill_entries( } }; - skills - .into_iter() - .map(skill_snapshot_entry_from_skill_info) - .collect() + openbitfun_agent_runtime::skills::filter_implicit_image_skills( + skills, + image_generation_available, + ) + .into_iter() + .map(skill_snapshot_entry_from_skill_info) + .collect() } fn skill_snapshot_entry_from_skill_info(skill: SkillInfo) -> SkillSnapshotEntry { diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/skills/catalog.rs b/src/crates/assembly/core/src/agentic/tools/implementations/skills/catalog.rs index 11d3e1fcea..a31d857990 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/skills/catalog.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/skills/catalog.rs @@ -21,6 +21,7 @@ mod tests { ); assert_eq!(builtin_skill_group_key("commit-push-pr"), Some("meta")); assert_eq!(builtin_skill_group_key("find-skills"), Some("meta")); + assert_eq!(builtin_skill_group_key("generate-image"), Some("creation")); assert_eq!(builtin_skill_group_key("debug"), Some("debugging")); assert_eq!(builtin_skill_group_key("multitask"), Some("coordination")); assert_eq!(builtin_skill_group_key("plan"), Some("planning")); diff --git a/src/crates/execution/agent-runtime/src/skills/catalog.rs b/src/crates/execution/agent-runtime/src/skills/catalog.rs index c232cc91d6..4eaa4a7a71 100644 --- a/src/crates/execution/agent-runtime/src/skills/catalog.rs +++ b/src/crates/execution/agent-runtime/src/skills/catalog.rs @@ -86,6 +86,10 @@ pub(super) const BUILTIN_SKILL_SPECS: &[BuiltinSkillSpec] = &[ dir_name: "openbitfun-frontend-dev", group: BuiltinSkillGroup::Creation, }, + BuiltinSkillSpec { + dir_name: "generate-image", + group: BuiltinSkillGroup::Creation, + }, BuiltinSkillSpec { dir_name: "ppt-design", group: BuiltinSkillGroup::Office, diff --git a/src/crates/execution/agent-runtime/src/skills/mod.rs b/src/crates/execution/agent-runtime/src/skills/mod.rs index 572e697edd..49406af68f 100644 --- a/src/crates/execution/agent-runtime/src/skills/mod.rs +++ b/src/crates/execution/agent-runtime/src/skills/mod.rs @@ -39,10 +39,10 @@ pub use roots::{ #[cfg(feature = "agent-runtime")] pub use selection::{ annotate_shadowed_skills, build_mode_skill_infos, filter_candidates_for_mode, - filter_implicitly_invocable_skills, filter_user_invocable_skills, is_skill_globally_enabled, - normalize_skill_keys, resolve_default_hidden_builtin_for_explicit_invocation, - resolve_visible_skills, sort_skill_candidates_by_dir, sort_skills, - ExplicitSkillInvocationResolution, SkillCandidate, + filter_implicit_image_skills, filter_implicitly_invocable_skills, filter_user_invocable_skills, + is_skill_globally_enabled, normalize_skill_keys, + resolve_default_hidden_builtin_for_explicit_invocation, resolve_visible_skills, + sort_skill_candidates_by_dir, sort_skills, ExplicitSkillInvocationResolution, SkillCandidate, }; pub use types::{ render_loaded_skill_for_assistant, ModeSkillInfo, ModeSkillStateReason, SkillData, diff --git a/src/crates/execution/agent-runtime/src/skills/policy.rs b/src/crates/execution/agent-runtime/src/skills/policy.rs index 86bebdf2c1..c15ecf1e0f 100644 --- a/src/crates/execution/agent-runtime/src/skills/policy.rs +++ b/src/crates/execution/agent-runtime/src/skills/policy.rs @@ -126,6 +126,11 @@ const ENABLE_PLAN: SkillPolicyRule = SkillPolicyRule { effect: PolicyEffect::Enable, }; +const ENABLE_GENERATE_IMAGE: SkillPolicyRule = SkillPolicyRule { + selector: SkillSelector::DirName("generate-image"), + effect: PolicyEffect::Enable, +}; + const OPEN_META_ONLY_POLICY: ModeSkillPolicy = ModeSkillPolicy { builtin_default: PolicyEffect::Disable, rules: &[ENABLE_META], @@ -139,6 +144,7 @@ const AGENTIC_POLICY: ModeSkillPolicy = ModeSkillPolicy { DISABLE_MINIAPP, DISABLE_CREATION, DISABLE_CANVAS, + ENABLE_GENERATE_IMAGE, ], }; @@ -151,6 +157,7 @@ const CLAW_POLICY: ModeSkillPolicy = ModeSkillPolicy { DISABLE_CREATION, DISABLE_DEBUGGING, DISABLE_CANVAS, + ENABLE_GENERATE_IMAGE, ], }; @@ -171,12 +178,13 @@ const COWORK_POLICY: ModeSkillPolicy = ModeSkillPolicy { ENABLE_META, ENABLE_COORDINATION, ENABLE_PLANNING, + ENABLE_GENERATE_IMAGE, ], }; const DEEP_RESEARCH_POLICY: ModeSkillPolicy = ModeSkillPolicy { builtin_default: PolicyEffect::Disable, - rules: &[ENABLE_META, ENABLE_COORDINATION], + rules: &[ENABLE_META, ENABLE_COORDINATION, ENABLE_GENERATE_IMAGE], }; const ULTRA_POLICY: ModeSkillPolicy = ModeSkillPolicy { @@ -234,6 +242,22 @@ mod tests { use crate::agents::{STANDARD_HARNESS_CONFIG_ID, STANDARD_HARNESS_CONFIG_MEMBERS}; use crate::skills::catalog::BUILTIN_SKILL_SPECS; + #[test] + fn image_generation_defaults_on_in_main_workflows_and_remains_opt_in_for_specialist_agents() { + for mode in ["Standard", "Claw", "Creative", "Cowork", "DeepResearch"] { + assert_eq!( + resolve_builtin_default_enabled("generate-image", mode), + Some(true) + ); + } + for mode in ["Ultimate", "SwarmWorker", "ComputerUse", "SomeUnknownMode"] { + assert_eq!( + resolve_builtin_default_enabled("generate-image", mode), + Some(false) + ); + } + } + #[test] fn agent_browser_defaults_on_only_for_ultra_and_swarm_worker() { for mode_id in [ diff --git a/src/crates/execution/agent-runtime/src/skills/selection.rs b/src/crates/execution/agent-runtime/src/skills/selection.rs index 9bd3ededc5..9538c1980b 100644 --- a/src/crates/execution/agent-runtime/src/skills/selection.rs +++ b/src/crates/execution/agent-runtime/src/skills/selection.rs @@ -3,6 +3,7 @@ use super::keys::normalize_skill_keys as normalize_skill_key_list; use super::resolver::{ resolve_skill_default_enabled_for_mode, resolve_skill_state_for_mode, UserModeSkillOverrides, }; +use super::roots::OPENBITFUN_USER_SKILL_SLOT; use super::types::{ModeSkillInfo, SkillData, SkillInfo, SkillLocation}; use std::collections::{HashMap, HashSet}; @@ -161,6 +162,27 @@ pub fn filter_user_invocable_skills(skills: Vec) -> Vec { .collect() } +/// Select the implicit image workflow after mode/user policy and tool availability. +/// Explicit invocation and the installed skill inventory retain all records. +pub fn filter_implicit_image_skills( + mut skills: Vec, + image_generation_available: bool, +) -> Vec { + let is_native = |skill: &SkillInfo| skill.is_builtin && skill.dir_name == "generate-image"; + let prefer_native = image_generation_available && skills.iter().any(is_native); + skills.retain(|skill| { + if is_native(skill) { + return image_generation_available; + } + !(prefer_native + && skill.name == "imagegen" + && skill.level == SkillLocation::User + && !skill.is_builtin + && skill.source_slot != OPENBITFUN_USER_SKILL_SLOT) + }); + skills +} + pub fn is_skill_globally_enabled( skill: &SkillInfo, globally_disabled_user_skills: &HashSet, @@ -303,6 +325,81 @@ pub fn resolve_default_hidden_builtin_for_explicit_invocation( mod tests { use super::*; + fn native_image_skill() -> SkillInfo { + let mut skill = candidate( + "generate-image", + "user::openbitfun-system::generate-image", + "/native/generate-image", + ) + .info; + skill.is_builtin = true; + skill.level = SkillLocation::User; + skill.source_slot = "openbitfun-system".to_string(); + skill + } + + fn external_image_skill() -> SkillInfo { + let mut skill = candidate( + "imagegen", + "user::home.codex::.system/imagegen", + "/external/imagegen", + ) + .info; + skill.level = SkillLocation::User; + skill.source_slot = "home.codex".to_string(); + skill + } + + #[test] + fn native_image_route_replaces_only_external_user_imagegen_in_implicit_listing() { + let mut user_authored = external_image_skill(); + user_authored.source_slot = OPENBITFUN_USER_SKILL_SLOT.to_string(); + user_authored.key = "user::openbitfun::imagegen".to_string(); + let project = candidate("imagegen", "project::codex::imagegen", "/project/imagegen").info; + let unrelated = candidate( + "brandkit", + "user::home.agents::brandkit", + "/external/brandkit", + ) + .info; + let original = vec![ + native_image_skill(), + external_image_skill(), + user_authored.clone(), + project.clone(), + unrelated.clone(), + ]; + let selected = filter_implicit_image_skills(original.clone(), true); + assert_eq!( + selected + .iter() + .map(|skill| skill.key.as_str()) + .collect::>(), + [ + original[0].key.as_str(), + user_authored.key.as_str(), + project.key.as_str(), + unrelated.key.as_str() + ] + ); + assert_eq!(filter_user_invocable_skills(original).len(), 5); + } + + #[test] + fn native_image_skill_is_hidden_when_its_tool_is_unavailable() { + let selected = + filter_implicit_image_skills(vec![native_image_skill(), external_image_skill()], false); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].key, external_image_skill().key); + } + + #[test] + fn opting_out_of_the_native_skill_keeps_the_external_image_route() { + let selected = filter_implicit_image_skills(vec![external_image_skill()], true); + assert_eq!(selected.len(), 1); + assert_eq!(selected[0].key, external_image_skill().key); + } + #[test] fn resolved_candidate_sort_is_case_insensitive_ascending_and_stable_for_equal_precedence() { let beta_first = candidate("beta", "project::beta", "/tmp/first"); From 1b3d949c28f6d57c0f3db2d2853932e569401ff4 Mon Sep 17 00:00:00 2001 From: kev1n77 Date: Mon, 26 Oct 2026 17:27:14 +0800 Subject: [PATCH 3/3] feat(images): simplify setup and show compact chat previews --- .../rust-build-dependency-boundaries.md | 4 + docs/interactive-capabilities/README.md | 16 +-- .../capabilities.json | 106 ++++++++++++++++-- .../capabilities/feature.ai-assistant.md | 2 + .../capabilities/setting.ai.models.md | 4 +- .../technical/product-control-open-audit.json | 11 +- .../technical/tauri-command-map.json | 24 +++- scripts/check-core-boundaries.test.mjs | 34 ++++++ .../cargo-dependency-boundaries.mjs | 2 + .../core-boundaries/rules/feature-rules.mjs | 28 ++++- scripts/core-boundaries/self-test.mjs | 1 + scripts/interactive-capabilities.test.mjs | 1 + src/apps/cli/Cargo.toml | 1 + src/apps/sdk-host/Cargo.toml | 1 + src/crates/assembly/core/AGENTS.md | 6 +- src/crates/assembly/core/Cargo.toml | 4 + src/crates/assembly/core/src/agentic/mod.rs | 2 +- .../implementations/generate_image_tool.rs | 20 ++++ .../src/agentic/tools/implementations/mod.rs | 4 +- .../tools/product_runtime/materialization.rs | 51 ++++++++- .../assembly/product-capabilities/src/lib.rs | 4 + .../product_capabilities.rs | 2 + .../generated/product-control-catalog.json | 106 ++++++++++++++++-- .../generated/remote-surface-registry.json | 14 ++- .../execution/tool-provider-groups/Cargo.toml | 3 +- .../execution/tool-provider-groups/src/lib.rs | 42 ++++++- src/crates/interfaces/acp/Cargo.toml | 1 + src/crates/services/services-core/AGENTS.md | 1 + .../interactive-capabilities/catalog.json | 46 +++++++- .../generated/interactive-capabilities.json | 106 ++++++++++++++++-- ...nteractiveCapabilitySearchProvider.test.ts | 11 ++ .../ai/ModelSettingsPage.loading.test.tsx | 21 +++- .../flow_chat/components/FlowTextBlock.tsx | 1 + .../components/voice/ControlConversation.tsx | 1 + .../api/generated/productControl.ts | 2 +- .../api/generated/remoteSurface.ts | 2 +- .../ImageGenerationEditor.tsx | 48 +------- .../ImageGenerationSection.scss | 4 +- .../ImageGenerationSettings.test.tsx | 28 ++++- .../image-generation/imageDialects.ts | 2 +- .../src/infrastructure/markdown/Markdown.scss | 49 ++++++++ .../markdown/MarkdownMathRenderer.tsx | 5 +- .../markdown/MarkdownRenderer.test.tsx | 59 ++++++++++ .../markdown/MarkdownRenderer.tsx | 23 +++- .../markdown/SessionMarkdownImage.tsx | 10 +- .../markdown/rehypeImageThumbnails.ts | 65 +++++++++++ .../locales/en-US/scenes/capabilities.json | 3 +- .../locales/zh-CN/scenes/capabilities.json | 3 +- .../locales/zh-TW/scenes/capabilities.json | 3 +- 49 files changed, 864 insertions(+), 123 deletions(-) create mode 100644 src/web-ui/src/infrastructure/markdown/rehypeImageThumbnails.ts diff --git a/docs/architecture/rust-build-dependency-boundaries.md b/docs/architecture/rust-build-dependency-boundaries.md index 770d082c91..ea5487fd96 100644 --- a/docs/architecture/rust-build-dependency-boundaries.md +++ b/docs/architecture/rust-build-dependency-boundaries.md @@ -82,6 +82,10 @@ Plugin Source 和完整 domain feature 集合一起带回 Agent Runtime。产品 编译”的 owner 返回类型化错误。`agent-runtime` 基线计划只选择 `Basic` 与 `AgentControl`； 它不是隐式 Delivery Profile，也不得从当前二进制已编译的 feature union 反推产品能力。 +图像查看与分析由 `tools-image-analysis` 独立选择；`tools-image-generation` 承载生成工具、 +配置驱动的 AI client 和生图协议适配。检查图片的窄消费者不应因此启用生成请求。完整产品和 +需要生图的 CLI、ACP、SDK Host 显式选择这两个 owner，工具 provider 与注册顺序保持稳定。 + ### 3.3 Workspace dependency 只提供共同底座 - workspace 声明负责版本和真正跨产品共享的最小 feature； diff --git a/docs/interactive-capabilities/README.md b/docs/interactive-capabilities/README.md index 057976a136..0b898d30a2 100644 --- a/docs/interactive-capabilities/README.md +++ b/docs/interactive-capabilities/README.md @@ -1,9 +1,9 @@ # OpenBitFun 功能与设置目录 / OpenBitFun Features & Settings -OpenBitFun Playbook 当前包含 **23 个功能**和 **21 个设置页**，共 **44 个**用户可理解的条目、**333 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 +OpenBitFun Playbook 当前包含 **23 个功能**和 **21 个设置页**，共 **44 个**用户可理解的条目、**334 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 -OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, and **333** source-backed sub-capabilities across **44** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. +OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, and **334** source-backed sub-capabilities across **44** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. ## 唯一事实源 / Single source of truth @@ -27,20 +27,20 @@ OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, a - Generated per-item interaction audit: `docs/interactive-capabilities/technical/product-control-open-audit.json` - Generated low-level audit map: `docs/interactive-capabilities/technical/tauri-command-map.json` -说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **701** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 +说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **702** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 -Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **701** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. +Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **702** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. ## 控制边界 / Control boundary -- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **52**、委托 **65**、需交互 **216**、不支持 **0**。 +- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **52**、委托 **66**、需交互 **216**、不支持 **0**。 - 稳定行为声明为带 JSON 输入契约的 `operations` 或 `options`，并绑定原生产品控制 Provider；智能体不接触原始 Tauri Command。 -- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 333 项子能力都不会写入 system prompt。 +- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 334 项子能力都不会写入 system prompt。 - 目录发现与契约读取不依赖 React 或可见窗口。普通配置型 option 统一由 Product Assembly 的共享 ConfigService 执行器读、写并回读，因此 Desktop、CLI 与 Headless 表面走同一份实现；只有宿主原生 operation/provider option 和界面导航按表面注册适配器，缺失时必须明确返回不可用，禁止静默回退本机。只读智能体只能发现和读取目录。 -- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **52 direct**, **65 delegated**, **216 interactive**, and **0 unsupported**. +- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **52 direct**, **66 delegated**, **216 interactive**, and **0 unsupported**. - Stable behavior becomes a typed `operation` or `option` with a JSON input contract and a native product-control provider. Agents never receive raw Tauri commands. -- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 333 documented items enters the system prompt. +- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 334 documented items enters the system prompt. - Discovery and contract lookup do not depend on React or a visible window. Ordinary config-backed options are read, written, and read back by one Product Assembly ConfigService executor shared by Desktop, CLI, and headless surfaces. Only host-native operations/provider options and presentation routes install surface adapters; missing adapters return explicit unavailability without local fallback. Read-only agents may only discover and inspect entries. ## 防腐化门禁 / Anti-drift gates diff --git a/docs/interactive-capabilities/capabilities.json b/docs/interactive-capabilities/capabilities.json index 5fde780727..096cc2e233 100644 --- a/docs/interactive-capabilities/capabilities.json +++ b/docs/interactive-capabilities/capabilities.json @@ -4,9 +4,21 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ + { + "id": "native-image-generation", + "query": "generate_image", + "expectedFirstCapabilityId": "feature.ai-assistant", + "expectedCapabilityIds": [ + "feature.ai-assistant" + ], + "expectedItem": { + "capabilityId": "feature.ai-assistant", + "itemId": "image-generation" + } + }, { "id": "companion-pet-mixed", "query": "宠物 pet mascot 桌面宠物", @@ -147,10 +159,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -202,6 +214,7 @@ "session-lifecycle", "model-mode-permissions", "creative-frontend-authoring", + "image-generation", "turn-control", "permission-mailbox", "history-search", @@ -443,6 +456,51 @@ "sceneId": "session" } }, + { + "id": "feature.ai-assistant:delegate:image-generation", + "capabilityId": "feature.ai-assistant", + "itemIds": [ + "image-generation" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "generate_image" + ], + "presentationTarget": { + "kind": "scene", + "sceneId": "session" + } + }, { "id": "feature.ai-assistant:open:turn-control", "capabilityId": "feature.ai-assistant", @@ -19823,6 +19881,27 @@ ] } }, + { + "id": "image-generation", + "titleZh": "通过已配置的默认图像模型生成图片", + "titleEn": "Generate images with the configured default image model", + "control": { + "kind": "delegate", + "tools": [ + "generate_image" + ], + "workflowZh": [ + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务" + ], + "workflowEn": [ + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services" + ] + } + }, { "id": "turn-control", "titleZh": "开始、引导、打断、取消或恢复一次智能体运行", @@ -20085,6 +20164,8 @@ "Choose the model, work mode, memory mode, and permission mode per session", "让创造模式扩展客户端界面与运行时能力", "Extend client UI and runtime capabilities in Creative mode", + "通过已配置的默认图像模型生成图片", + "Generate images with the configured default image model", "开始、引导、打断、取消或恢复一次智能体运行", "Start, steer, interrupt, cancel, or recover an agent turn", "查看并逐个或批量回答等待中的权限请求", @@ -20125,6 +20206,13 @@ "Call FrontendWorkbench prepare on the local Desktop in Creative mode, read the packaged API, and edit only draft CSS, JavaScript and owned assets without source or build tools", "Apply with the original draft ID; the host starts the immutable review window's 15-second countdown only after the real shell and customization activate, and apply waits for a final confirmed or rolled_back outcome", "Compose commands, state and events into reusable capabilities; use FrontendWorkbench inspect to discover schemas and diagnostics, then invoke to verify execution", + "generate_image", + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务", + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services", "ListWorkspaces", "SessionControl", "SessionHistory", @@ -27559,13 +27647,13 @@ }, { "id": "connection-test", - "titleZh": "保存前测试供应商或模型连接并刷新运行时客户端", - "titleEn": "Test provider or model connectivity before saving and refresh the runtime client", + "titleZh": "测试模型连接或图像生成，并刷新运行时客户端", + "titleEn": "Test model connections or image generation and refresh the runtime client", "control": { "kind": "open", "reasonCode": "unstructuredInteraction", - "reasonZh": "“保存前测试供应商或模型连接并刷新运行时客户端”由多个实时状态相关步骤组成，目前没有一个能确定完成整个流程的单一结构化 Command；智能体会打开精确入口，并把后续交互保留在用户可见界面。", - "reasonEn": "“Test provider or model connectivity before saving and refresh the runtime client” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user." + "reasonZh": "“测试模型连接或图像生成，并刷新运行时客户端”需要用户在配置表单中选择服务并主动发起；图像服务测试会实际生成图片并可能消耗额度，智能体只能打开对应设置入口。", + "reasonEn": "“Test model connections or image generation and refresh the runtime client” requires the user to select a configuration and start the test from its form; image-service tests generate an actual image and may consume quota, so the Agent can only open the settings entry." } }, { @@ -27679,8 +27767,8 @@ "Inspect models.dev catalog status, refresh it manually, and reveal its local cache", "为供应商添加自定义 Header、请求体字段与 SSL 行为", "Add custom headers, request-body fields, and SSL behavior per provider", - "保存前测试供应商或模型连接并刷新运行时客户端", - "Test provider or model connectivity before saving and refresh the runtime client", + "测试模型连接或图像生成，并刷新运行时客户端", + "Test model connections or image generation and refresh the runtime client", "管理模型池、路由策略与会话模型选择", "Manage model pools, routing strategies, and session model selection", "打开模型设置", diff --git a/docs/interactive-capabilities/capabilities/feature.ai-assistant.md b/docs/interactive-capabilities/capabilities/feature.ai-assistant.md index 54ad8079ec..d2aea4e449 100644 --- a/docs/interactive-capabilities/capabilities/feature.ai-assistant.md +++ b/docs/interactive-capabilities/capabilities/feature.ai-assistant.md @@ -25,6 +25,8 @@ Work with AI in project context, create sessions, continue tasks, answer permiss - Choose the model, work mode, memory mode, and permission mode per session - **由专用智能体工具控制 / Delegated Agent tool** · `FrontendWorkbench` / `Read` / `Write` / `Edit` · 让创造模式扩展客户端界面与运行时能力 - Extend client UI and runtime capabilities in Creative mode +- **由专用智能体工具控制 / Delegated Agent tool** · `generate_image` · 通过已配置的默认图像模型生成图片 + - Generate images with the configured default image model - **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 开始、引导、打断、取消或恢复一次智能体运行 - Start, steer, interrupt, cancel, or recover an agent turn - **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 查看并逐个或批量回答等待中的权限请求 diff --git a/docs/interactive-capabilities/capabilities/setting.ai.models.md b/docs/interactive-capabilities/capabilities/setting.ai.models.md index 55e2b6ee37..198f5ff42f 100644 --- a/docs/interactive-capabilities/capabilities/setting.ai.models.md +++ b/docs/interactive-capabilities/capabilities/setting.ai.models.md @@ -37,8 +37,8 @@ Add AI providers and models, choose defaults, and configure proxy, subscription, - Inspect models.dev catalog status, refresh it manually, and reveal its local cache - **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 为供应商添加自定义 Header、请求体字段与 SSL 行为 - Add custom headers, request-body fields, and SSL behavior per provider -- **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 保存前测试供应商或模型连接并刷新运行时客户端 - - Test provider or model connectivity before saving and refresh the runtime client +- **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 测试模型连接或图像生成，并刷新运行时客户端 + - Test model connections or image generation and refresh the runtime client - **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 管理模型池、路由策略与会话模型选择 - Manage model pools, routing strategies, and session model selection diff --git a/docs/interactive-capabilities/technical/product-control-open-audit.json b/docs/interactive-capabilities/technical/product-control-open-audit.json index 602eccb317..6efb101c0b 100644 --- a/docs/interactive-capabilities/technical/product-control-open-audit.json +++ b/docs/interactive-capabilities/technical/product-control-open-audit.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "catalogDigest": "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77", "count": 216, "reasonCounts": { "externalAuth": 4, @@ -2862,11 +2862,11 @@ { "capabilityId": "setting.ai.models", "itemId": "connection-test", - "titleZh": "保存前测试供应商或模型连接并刷新运行时客户端", - "titleEn": "Test provider or model connectivity before saving and refresh the runtime client", + "titleZh": "测试模型连接或图像生成，并刷新运行时客户端", + "titleEn": "Test model connections or image generation and refresh the runtime client", "reasonCode": "unstructuredInteraction", - "reasonZh": "“保存前测试供应商或模型连接并刷新运行时客户端”由多个实时状态相关步骤组成，目前没有一个能确定完成整个流程的单一结构化 Command；智能体会打开精确入口，并把后续交互保留在用户可见界面。", - "reasonEn": "“Test provider or model connectivity before saving and refresh the runtime client” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user.", + "reasonZh": "“测试模型连接或图像生成，并刷新运行时客户端”需要用户在配置表单中选择服务并主动发起；图像服务测试会实际生成图片并可能消耗额度，智能体只能打开对应设置入口。", + "reasonEn": "“Test model connections or image generation and refresh the runtime client” requires the user to select a configuration and start the test from its form; image-service tests generate an actual image and may consume quota, so the Agent can only open the settings entry.", "presentationTarget": { "kind": "settings", "pageId": "ai.models" @@ -2874,6 +2874,7 @@ "evidence": [ "command:test_ai_connection", "command:test_ai_config_connection", + "command:test_image_generation_service", "command:refresh_model_client" ] }, diff --git a/docs/interactive-capabilities/technical/tauri-command-map.json b/docs/interactive-capabilities/technical/tauri-command-map.json index 8e7c9ceeea..d27aa826c5 100644 --- a/docs/interactive-capabilities/technical/tauri-command-map.json +++ b/docs/interactive-capabilities/technical/tauri-command-map.json @@ -1,11 +1,11 @@ { "schemaVersion": 2, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", - "commandCount": 701, + "catalogDigest": "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77", + "commandCount": 702, "coverage": { - "commandCount": 701, - "documentedCommandCount": 644, + "commandCount": 702, + "documentedCommandCount": 645, "implementationCommandCount": 57, "implementationDigest": "503b48eebc37750e50e0427d861e884dfa353a81aae32236e4bc2d7e0f5310e2" }, @@ -10661,6 +10661,22 @@ "signature": "fn test_ai_connection(state: State<'_, AppState>) -> Result", "remoteWorkspacePolicy": "LegacyUnaudited" }, + { + "id": "test_image_generation_service", + "moduleId": "commands", + "capabilityId": "setting.ai.models", + "capabilityIds": [ + "setting.ai.models" + ], + "documentedItemIds": [ + "setting.ai.models:connection-test" + ], + "visibility": "documented", + "rustPath": "test_image_generation_service", + "sourceFile": "src/apps/desktop/src/api/commands.rs", + "signature": "fn test_image_generation_service( request: TestImageGenerationServiceRequest, ) -> Result", + "remoteWorkspacePolicy": "WorkspaceAgnostic" + }, { "id": "toggle_main_window_fullscreen", "moduleId": "system", diff --git a/scripts/check-core-boundaries.test.mjs b/scripts/check-core-boundaries.test.mjs index c2c521db49..96efc90eab 100644 --- a/scripts/check-core-boundaries.test.mjs +++ b/scripts/check-core-boundaries.test.mjs @@ -730,6 +730,38 @@ test('Services Core text runtime features keep independent exact owner profiles' ]); }); +test('image inspection and generation keep independent exact feature closures', async () => { + const manifestPath = 'src/crates/assembly/core/Cargo.toml'; + const manifest = await readFile(new URL(`../${manifestPath}`, import.meta.url), 'utf8'); + const features = parseManifestFeatures(manifest); + for (const [name, refs] of [ + ['tools-image-analysis', ['openbitfun-tool-packs/image-analysis']], + ['tools-image-generation', [ + 'openbitfun-tool-packs/image-generation', + 'ai-adapter-runtime', + 'openbitfun-ai-adapters/image-generation', + ]], + ]) { + const rule = coreClosedFeatureProfileRules.find(candidate => candidate.manifestPath === manifestPath && candidate.featureName === name); + assert.equal(rule?.exact, true); + assert.deepEqual(rule.requiredFeatureRefs, refs); + assert.deepEqual(features[name], refs); + } +}); + +test('system proxy discovery owns registry access without process or storage capabilities', async () => { + const manifestPath = 'src/crates/services/services-core/Cargo.toml'; + const manifest = await readFile(new URL(`../${manifestPath}`, import.meta.url), 'utf8'); + const expected = ['dep:windows', 'windows/Win32_Foundation', 'windows/Win32_System_Registry']; + assert.deepEqual(parseManifestFeatures(manifest)['system-proxy'], expected); + const rule = coreClosedFeatureProfileRules.find(candidate => candidate.manifestPath === manifestPath && candidate.featureName === 'system-proxy'); + assert.equal(rule?.exact, true); + assert.deepEqual(rule.requiredFeatureRefs, expected); + const windows = optionalDependencyFeatureOwnerRules.find(candidate => candidate.crateName === 'services-core') + .dependencies.find(dependency => dependency.depName === 'windows'); + assert.ok(windows.ownerFeatures.includes('system-proxy')); +}); + function parseManifestFeatures(manifest) { const section = manifest.match(/^\[features\]\s*$([\s\S]*?)(?=^\[|(?![\s\S]))/m)?.[1] ?? ''; const features = {}; @@ -1494,6 +1526,7 @@ test('Core product-full explicitly assembles service and tool capability owners' 'tools-computer-use', 'tools-creation', 'tools-image-analysis', + 'tools-image-generation', 'tools-miniapp', 'tools-canvas', 'tools-agent-control', @@ -1867,6 +1900,7 @@ const SDK_HOST_REVIEWED_CORE_FEATURES = [ 'tools-browser-web', 'tools-computer-use', 'tools-image-analysis', + 'tools-image-generation', 'tools-agent-control', ]; diff --git a/scripts/core-boundaries/cargo-dependency-boundaries.mjs b/scripts/core-boundaries/cargo-dependency-boundaries.mjs index 4503bbf61d..eb1a6863a8 100644 --- a/scripts/core-boundaries/cargo-dependency-boundaries.mjs +++ b/scripts/core-boundaries/cargo-dependency-boundaries.mjs @@ -1455,6 +1455,7 @@ export function findProductEntrypointCoreFeatureViolations( 'tools-browser-web', 'tools-computer-use', 'tools-image-analysis', + 'tools-image-generation', 'tools-agent-control', ]; const CORE_TEST_SUPPORT_FEATURE = 'test-support'; @@ -1512,6 +1513,7 @@ export function findProductEntrypointCoreFeatureViolations( 'tools-creation', 'tools-git', 'tools-image-analysis', + 'tools-image-generation', 'tools-mcp', 'tools-miniapp', 'web-tools', diff --git a/scripts/core-boundaries/rules/feature-rules.mjs b/scripts/core-boundaries/rules/feature-rules.mjs index 615a352a3a..e8f7da069b 100644 --- a/scripts/core-boundaries/rules/feature-rules.mjs +++ b/scripts/core-boundaries/rules/feature-rules.mjs @@ -96,7 +96,7 @@ export const optionalDependencyFeatureOwnerRules = [ }, { depName: 'which', ownerFeatures: ['process-runtime'] }, { depName: 'win32job', ownerFeatures: ['process-runtime'] }, - { depName: 'windows', ownerFeatures: ['installed-apps', 'json-io', 'local-storage', 'process-runtime'] }, + { depName: 'windows', ownerFeatures: ['installed-apps', 'json-io', 'local-storage', 'process-runtime', 'system-proxy'] }, { depName: 'tokio', ownerFeatures: [ @@ -166,7 +166,7 @@ export const optionalDependencyFeatureOwnerRules = [ { depName: 'base64', ownerFeatures: ['agent-runtime', 'dispatch-store'] }, { depName: 'openbitfun-ai-adapters', - ownerFeatures: ['ai-adapter-runtime', 'subscription-auth'], + ownerFeatures: ['ai-adapter-runtime', 'subscription-auth', 'tools-image-generation'], }, { depName: 'openbitfun-agent-runtime', ownerFeatures: ['agent-runtime', 'ai-adapter-runtime'] }, { depName: 'openbitfun-agent-workflows', ownerFeatures: ['deep-research'] }, @@ -232,6 +232,7 @@ export const optionalDependencyFeatureOwnerRules = [ 'tools-creation', 'tools-git', 'tools-image-analysis', + 'tools-image-generation', 'tools-mcp', 'tools-miniapp', 'tools-pages', @@ -691,7 +692,7 @@ export const capabilityContractDependencyRules = [ capabilityForwarder('ai-adapter-runtime', 'account-routing'), ], ['agent-runtime', 'ai-adapter-runtime'], - ['external-sources', 'function-agents', 'model-catalog', 'mcp-runtime', 'opencode-plugin-host', 'plugin-runtime', 'product-search', 'product-full', 'ptc-runtime', 'remote-connect', 'tools-mcp'], + ['external-sources', 'function-agents', 'model-catalog', 'mcp-runtime', 'opencode-plugin-host', 'plugin-runtime', 'product-search', 'product-full', 'ptc-runtime', 'remote-connect', 'tools-mcp', 'tools-image-generation'], )], ['openbitfun-desktop', capabilityConsumer([ capabilityEdge(['agent-runtime']), @@ -745,6 +746,7 @@ export const coreProductFullFeatureAssemblyRule = { 'tools-creation', 'tools-git', 'tools-image-analysis', + 'tools-image-generation', 'tools-mcp', 'tools-miniapp', 'tools-pages', @@ -789,6 +791,7 @@ export const acpServerCoreFeatures = [ 'tools-browser-web', 'tools-computer-use', 'tools-image-analysis', + 'tools-image-generation', 'tools-agent-control', ]; @@ -1381,6 +1384,17 @@ export const coreClosedFeatureProfileRules = [ exact: true, reason: 'tools-image-analysis must own only explicit image inspection tools', }, + { + manifestPath: 'src/crates/assembly/core/Cargo.toml', + featureName: 'tools-image-generation', + requiredFeatureRefs: [ + 'openbitfun-tool-packs/image-generation', + 'ai-adapter-runtime', + 'openbitfun-ai-adapters/image-generation', + ], + exact: true, + reason: 'tools-image-generation owns image generation with its configured AI client and wire adapter', + }, { manifestPath: 'src/crates/assembly/core/Cargo.toml', featureName: 'tools-pages', @@ -1473,6 +1487,13 @@ export const coreClosedFeatureProfileRules = [ exact: true, reason: 'services-core TLS provider owner must select only the workspace Rustls implementation', }, + { + manifestPath: 'src/crates/services/services-core/Cargo.toml', + featureName: 'system-proxy', + requiredFeatureRefs: ['dep:windows', 'windows/Win32_Foundation', 'windows/Win32_System_Registry'], + exact: true, + reason: 'services-core system proxy discovery owns only Windows registry access and environment parsing', + }, { manifestPath: 'src/crates/services/services-core/Cargo.toml', featureName: 'diagnostics', @@ -1817,6 +1838,7 @@ export const ownerCrateFeatureAssemblyRules = [ 'browser-web', 'computer-use', 'image-analysis', + 'image-generation', 'miniapp', 'pages', 'creation', diff --git a/scripts/core-boundaries/self-test.mjs b/scripts/core-boundaries/self-test.mjs index bf30084e0d..93804e6636 100644 --- a/scripts/core-boundaries/self-test.mjs +++ b/scripts/core-boundaries/self-test.mjs @@ -634,6 +634,7 @@ export function runManifestParserSelfTest({ 'openbitfun-core/tools-browser-web', 'openbitfun-core/tools-computer-use', 'openbitfun-core/tools-image-analysis', + 'openbitfun-core/tools-image-generation', 'openbitfun-core/tools-agent-control', ], ], diff --git a/scripts/interactive-capabilities.test.mjs b/scripts/interactive-capabilities.test.mjs index 8af06d4dc1..35eb784333 100644 --- a/scripts/interactive-capabilities.test.mjs +++ b/scripts/interactive-capabilities.test.mjs @@ -229,6 +229,7 @@ test('every product Agent tool is mapped to a user capability or explicitly clas const byId = new Map(source.capabilities.map((capability) => [capability.id, capability])); const item = (capabilityId, itemId) => byId.get(capabilityId)?.items .find(({ id }) => id === itemId); + assert.deepEqual(item('feature.ai-assistant', 'image-generation').control.tools, ['generate_image']); assert.ok(item('feature.remote-workspaces', 'remote-files').control.tools.includes('Read')); assert.deepEqual(item('feature.remote-workspaces', 'port-forwarding').control.tools, ['PortForward']); assert.ok(item('setting.tools.mcp', 'resources').control.tools.includes('ReadMCPResource')); diff --git a/src/apps/cli/Cargo.toml b/src/apps/cli/Cargo.toml index 5de4e6c447..bbdc803d8a 100644 --- a/src/apps/cli/Cargo.toml +++ b/src/apps/cli/Cargo.toml @@ -47,6 +47,7 @@ openbitfun-core = { path = "../../crates/assembly/core", features = [ "tools-browser-web", "tools-computer-use", "tools-image-analysis", + "tools-image-generation", "tools-agent-control", ] } openbitfun-events = { path = "../../crates/contracts/events" } diff --git a/src/apps/sdk-host/Cargo.toml b/src/apps/sdk-host/Cargo.toml index f1912b70f7..c5647c0cd1 100644 --- a/src/apps/sdk-host/Cargo.toml +++ b/src/apps/sdk-host/Cargo.toml @@ -25,6 +25,7 @@ openbitfun-core = { path = "../../crates/assembly/core", features = [ "tools-browser-web", "tools-computer-use", "tools-image-analysis", + "tools-image-generation", "tools-agent-control", ] } openbitfun-sdk-host = { path = "../../crates/interfaces/sdk-host" } diff --git a/src/crates/assembly/core/AGENTS.md b/src/crates/assembly/core/AGENTS.md index 4b8cfc7eac..e08ba7f80a 100644 --- a/src/crates/assembly/core/AGENTS.md +++ b/src/crates/assembly/core/AGENTS.md @@ -114,6 +114,8 @@ SessionManager -> Session -> DialogTurn -> ModelRound Function Agent adapters use the independent `function-agents` owner; MiniApp domain/runtime/market dependencies belong only to `tools-miniapp`. Tool implementation groups use the matching `tools-*` owner feature. + `tools-image-analysis` owns only image inspection; `tools-image-generation` + separately selects the generation tool and its AI adapter client. Product Assembly supplies the exact `ProductToolPlan`; Core materialization validates that requested owners were compiled and must not infer product capability from Cargo's feature union. The Agent Runtime baseline plan is @@ -213,7 +215,9 @@ cargo test -p openbitfun-core --no-default-features --features ai-adapter-runtim For image-service selection and compatibility with saved subscription configurations: ```bash -cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-image-analysis,subscription-auth --lib agentic::image_generation::tests +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-image-generation,subscription-auth --lib agentic::image_generation::tests +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-image-generation --lib agentic::tools::product_runtime::materialization::tests::image_tools_follow_independent_compiled_owners +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-image-analysis --lib agentic::tools::product_runtime::materialization::tests::image_tools_follow_independent_compiled_owners ``` These fixtures do not call a provider or exercise live remote hosts. Image wire diff --git a/src/crates/assembly/core/Cargo.toml b/src/crates/assembly/core/Cargo.toml index c4f1420803..e7bbc13bdc 100644 --- a/src/crates/assembly/core/Cargo.toml +++ b/src/crates/assembly/core/Cargo.toml @@ -170,6 +170,7 @@ product-full = [ "tools-browser-web", "tools-computer-use", "tools-image-analysis", + "tools-image-generation", "tools-miniapp", "tools-pages", "tools-creation", @@ -377,6 +378,9 @@ tools-computer-use = [ ] tools-image-analysis = [ "openbitfun-tool-packs/image-analysis", +] +tools-image-generation = [ + "openbitfun-tool-packs/image-generation", "ai-adapter-runtime", "openbitfun-ai-adapters/image-generation", ] diff --git a/src/crates/assembly/core/src/agentic/mod.rs b/src/crates/assembly/core/src/agentic/mod.rs index f2aff80509..4418e8e438 100644 --- a/src/crates/assembly/core/src/agentic/mod.rs +++ b/src/crates/assembly/core/src/agentic/mod.rs @@ -33,7 +33,7 @@ pub mod round_preempt; // Image analysis module pub mod image_analysis; -#[cfg(feature = "tools-image-analysis")] +#[cfg(feature = "tools-image-generation")] pub mod image_generation; pub(crate) mod keyed_lock; pub mod memories; diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs index a819564881..bd8cea018a 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs @@ -257,6 +257,26 @@ impl Tool for GenerateImageTool { has_enabled_image_service().await } + fn output_schema(&self) -> Value { + json!({ + "type": "object", + "properties": { + "path": {"type": "string", "description": "Path of the first generated image on the execution host."}, + "paths": {"type": "array", "items": {"type": "string"}, "minItems": 1}, + "count": {"type": "integer", "minimum": 1}, + "service_id": {"type": "string"}, + "service": {"type": "string"}, + "dialect": {"type": "string"}, + "model": {"type": "string"}, + "mime_type": {"type": "string"}, + "bytes": {"type": "integer", "minimum": 0, "description": "Byte length of the first generated image."}, + "summary": {"type": "string"} + }, + "required": ["path", "paths", "count", "service_id", "service", "dialect", "model", "mime_type", "bytes", "summary"], + "additionalProperties": true + }) + } + fn is_readonly(&self) -> bool { false } diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs index 3885dc7647..ecb2ec956f 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs @@ -36,7 +36,7 @@ pub mod file_read_tool; pub mod file_write_tool; #[cfg(feature = "tools-creation")] pub mod frontend_workbench_tool; -#[cfg(feature = "tools-image-analysis")] +#[cfg(feature = "tools-image-generation")] pub mod generate_image_tool; #[cfg(feature = "tools-miniapp")] pub mod generative_ui_tool; @@ -113,7 +113,7 @@ pub use file_read_tool::FileReadTool; pub use file_write_tool::FileWriteTool; #[cfg(feature = "tools-creation")] pub use frontend_workbench_tool::FrontendWorkbenchTool; -#[cfg(feature = "tools-image-analysis")] +#[cfg(feature = "tools-image-generation")] pub use generate_image_tool::GenerateImageTool; #[cfg(feature = "tools-miniapp")] pub use generative_ui_tool::GenerativeUITool; diff --git a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs index fa18983803..5d51558d68 100644 --- a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs +++ b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs @@ -112,7 +112,7 @@ impl StaticToolProviderFactory for ProductConcreteToolFactory { "view_image" => Some(Arc::new(ViewImageTool::new())), #[cfg(feature = "tools-image-analysis")] "analyze_image" => Some(Arc::new(AnalyzeImageTool::new())), - #[cfg(feature = "tools-image-analysis")] + #[cfg(feature = "tools-image-generation")] "generate_image" => Some(Arc::new(GenerateImageTool::new())), "Glob" => Some(Arc::new(GlobTool::new())), "Grep" => Some(Arc::new(GrepTool::new())), @@ -262,6 +262,55 @@ mod tests { use crate::agentic::tools::framework::ToolExposure; use openbitfun_tool_packs::product_tool_provider_group_plan; + #[test] + fn image_tools_follow_independent_compiled_owners() { + for (group, expected, compiled) in [ + ( + ToolPackFeatureGroup::ImageGeneration, + &["generate_image"][..], + cfg!(feature = "tools-image-generation"), + ), + ( + ToolPackFeatureGroup::ImageAnalysis, + &["view_image", "analyze_image"][..], + cfg!(feature = "tools-image-analysis"), + ), + ] { + let result = create_product_tool_registry_from_plan( + product_tool_provider_group_plan(), + &[group], + Arc::new(openbitfun_agent_tools::SnapshotToolDecorator::new( + Arc::new(super::super::snapshot::ProductSnapshotToolWrapper), + )), + ); + if !compiled { + assert!(matches!( + result, + Err(ProductToolMaterializationError::UnavailableFeatureGroups { .. }) + )); + continue; + } + let registry = result.expect("the explicitly selected image owner must materialize"); + assert_eq!( + registry + .get_tool_names() + .iter() + .map(String::as_str) + .collect::>(), + expected + ); + for tool in registry.get_all_tools() { + assert!( + tool.output_schema() + .as_object() + .is_some_and(|schema| !schema.is_empty()), + "{} must expose its output contract", + tool.name() + ); + } + } + } + /// Every tool name the provider plan advertises must materialize. /// /// Registry construction treats an unknown name as a hard error, so a tool diff --git a/src/crates/assembly/product-capabilities/src/lib.rs b/src/crates/assembly/product-capabilities/src/lib.rs index bc26b86ceb..0de5f1c7d3 100644 --- a/src/crates/assembly/product-capabilities/src/lib.rs +++ b/src/crates/assembly/product-capabilities/src/lib.rs @@ -58,6 +58,7 @@ pub enum ProductFeatureGroup { BrowserWeb, ComputerUse, ImageAnalysis, + ImageGeneration, MiniApp, Pages, Creation, @@ -74,6 +75,7 @@ impl ProductFeatureGroup { Self::BrowserWeb => "browser-web", Self::ComputerUse => "computer-use", Self::ImageAnalysis => "image-analysis", + Self::ImageGeneration => "image-generation", Self::MiniApp => "miniapp", Self::Pages => "pages", Self::Creation => "creation", @@ -98,6 +100,7 @@ impl From for ProductFeatureGroup { ToolPackFeatureGroup::BrowserWeb => Self::BrowserWeb, ToolPackFeatureGroup::ComputerUse => Self::ComputerUse, ToolPackFeatureGroup::ImageAnalysis => Self::ImageAnalysis, + ToolPackFeatureGroup::ImageGeneration => Self::ImageGeneration, ToolPackFeatureGroup::MiniApp => Self::MiniApp, ToolPackFeatureGroup::Pages => Self::Pages, ToolPackFeatureGroup::Creation => Self::Creation, @@ -116,6 +119,7 @@ impl From for ToolPackFeatureGroup { ProductFeatureGroup::BrowserWeb => Self::BrowserWeb, ProductFeatureGroup::ComputerUse => Self::ComputerUse, ProductFeatureGroup::ImageAnalysis => Self::ImageAnalysis, + ProductFeatureGroup::ImageGeneration => Self::ImageGeneration, ProductFeatureGroup::MiniApp => Self::MiniApp, ProductFeatureGroup::Pages => Self::Pages, ProductFeatureGroup::Creation => Self::Creation, diff --git a/src/crates/assembly/product-capabilities/tests/product_capability_contracts/product_capabilities.rs b/src/crates/assembly/product-capabilities/tests/product_capability_contracts/product_capabilities.rs index 27a574a61f..82dd42ef74 100644 --- a/src/crates/assembly/product-capabilities/tests/product_capability_contracts/product_capabilities.rs +++ b/src/crates/assembly/product-capabilities/tests/product_capability_contracts/product_capabilities.rs @@ -495,6 +495,7 @@ fn product_assembly_plan_exposes_build_feature_groups_explicitly() { &[ ProductFeatureGroup::Basic, ProductFeatureGroup::ImageAnalysis, + ProductFeatureGroup::ImageGeneration, ProductFeatureGroup::AgentControl, ProductFeatureGroup::Git, ProductFeatureGroup::BrowserWeb, @@ -511,6 +512,7 @@ fn product_assembly_plan_exposes_build_feature_groups_explicitly() { vec![ "basic", "image-analysis", + "image-generation", "agent-control", "git", "browser-web", diff --git a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json index f9849d0107..7454615ad9 100644 --- a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json +++ b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json @@ -4,9 +4,21 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ + { + "id": "native-image-generation", + "query": "generate_image", + "expectedFirstCapabilityId": "feature.ai-assistant", + "expectedCapabilityIds": [ + "feature.ai-assistant" + ], + "expectedItem": { + "capabilityId": "feature.ai-assistant", + "itemId": "image-generation" + } + }, { "id": "companion-pet-mixed", "query": "宠物 pet mascot 桌面宠物", @@ -147,10 +159,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -202,6 +214,7 @@ "session-lifecycle", "model-mode-permissions", "creative-frontend-authoring", + "image-generation", "turn-control", "permission-mailbox", "history-search", @@ -443,6 +456,51 @@ "sceneId": "session" } }, + { + "id": "feature.ai-assistant:delegate:image-generation", + "capabilityId": "feature.ai-assistant", + "itemIds": [ + "image-generation" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "generate_image" + ], + "presentationTarget": { + "kind": "scene", + "sceneId": "session" + } + }, { "id": "feature.ai-assistant:open:turn-control", "capabilityId": "feature.ai-assistant", @@ -19823,6 +19881,27 @@ ] } }, + { + "id": "image-generation", + "titleZh": "通过已配置的默认图像模型生成图片", + "titleEn": "Generate images with the configured default image model", + "control": { + "kind": "delegate", + "tools": [ + "generate_image" + ], + "workflowZh": [ + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务" + ], + "workflowEn": [ + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services" + ] + } + }, { "id": "turn-control", "titleZh": "开始、引导、打断、取消或恢复一次智能体运行", @@ -20089,6 +20168,8 @@ "Choose the model, work mode, memory mode, and permission mode per session", "让创造模式扩展客户端界面与运行时能力", "Extend client UI and runtime capabilities in Creative mode", + "通过已配置的默认图像模型生成图片", + "Generate images with the configured default image model", "开始、引导、打断、取消或恢复一次智能体运行", "Start, steer, interrupt, cancel, or recover an agent turn", "查看并逐个或批量回答等待中的权限请求", @@ -20129,6 +20210,13 @@ "Call FrontendWorkbench prepare on the local Desktop in Creative mode, read the packaged API, and edit only draft CSS, JavaScript and owned assets without source or build tools", "Apply with the original draft ID; the host starts the immutable review window's 15-second countdown only after the real shell and customization activate, and apply waits for a final confirmed or rolled_back outcome", "Compose commands, state and events into reusable capabilities; use FrontendWorkbench inspect to discover schemas and diagnostics, then invoke to verify execution", + "generate_image", + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务", + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services", "ListWorkspaces", "SessionControl", "SessionHistory", @@ -27786,13 +27874,13 @@ }, { "id": "connection-test", - "titleZh": "保存前测试供应商或模型连接并刷新运行时客户端", - "titleEn": "Test provider or model connectivity before saving and refresh the runtime client", + "titleZh": "测试模型连接或图像生成，并刷新运行时客户端", + "titleEn": "Test model connections or image generation and refresh the runtime client", "control": { "kind": "open", "reasonCode": "unstructuredInteraction", - "reasonZh": "“保存前测试供应商或模型连接并刷新运行时客户端”由多个实时状态相关步骤组成，目前没有一个能确定完成整个流程的单一结构化 Command；智能体会打开精确入口，并把后续交互保留在用户可见界面。", - "reasonEn": "“Test provider or model connectivity before saving and refresh the runtime client” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user." + "reasonZh": "“测试模型连接或图像生成，并刷新运行时客户端”需要用户在配置表单中选择服务并主动发起；图像服务测试会实际生成图片并可能消耗额度，智能体只能打开对应设置入口。", + "reasonEn": "“Test model connections or image generation and refresh the runtime client” requires the user to select a configuration and start the test from its form; image-service tests generate an actual image and may consume quota, so the Agent can only open the settings entry." } }, { @@ -27914,8 +28002,8 @@ "Inspect models.dev catalog status, refresh it manually, and reveal its local cache", "为供应商添加自定义 Header、请求体字段与 SSL 行为", "Add custom headers, request-body fields, and SSL behavior per provider", - "保存前测试供应商或模型连接并刷新运行时客户端", - "Test provider or model connectivity before saving and refresh the runtime client", + "测试模型连接或图像生成，并刷新运行时客户端", + "Test model connections or image generation and refresh the runtime client", "管理模型池、路由策略与会话模型选择", "Manage model pools, routing strategies, and session model selection", "打开模型设置", diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index 9e7b6826eb..de9d24a16c 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:eb8901b156df1dd3", + "digest": "fnv1a64:d64185f2013646ad", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -8191,6 +8191,18 @@ "reason": "the CLI peer host has no handler for this command" } }, + { + "id": "test_image_generation_service", + "surface": "tauri_command", + "remoteWorkspace": "WorkspaceAgnostic", + "peer": { + "kind": "proxied" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the CLI peer host has no handler for this command" + } + }, { "id": "toggle_main_window_fullscreen", "surface": "tauri_command", diff --git a/src/crates/execution/tool-provider-groups/Cargo.toml b/src/crates/execution/tool-provider-groups/Cargo.toml index 857da734d9..8cddcc4d2f 100644 --- a/src/crates/execution/tool-provider-groups/Cargo.toml +++ b/src/crates/execution/tool-provider-groups/Cargo.toml @@ -17,12 +17,13 @@ mcp = [] browser-web = [] computer-use = [] image-analysis = [] +image-generation = [] miniapp = [] pages = [] creation = [] canvas = [] agent-control = [] -product-full = ["basic", "git", "mcp", "browser-web", "computer-use", "image-analysis", "miniapp", "pages", "creation", "canvas", "agent-control"] +product-full = ["basic", "git", "mcp", "browser-web", "computer-use", "image-analysis", "image-generation", "miniapp", "pages", "creation", "canvas", "agent-control"] [lints] workspace = true diff --git a/src/crates/execution/tool-provider-groups/src/lib.rs b/src/crates/execution/tool-provider-groups/src/lib.rs index ca219da096..ac0cc7f1ef 100644 --- a/src/crates/execution/tool-provider-groups/src/lib.rs +++ b/src/crates/execution/tool-provider-groups/src/lib.rs @@ -15,6 +15,7 @@ pub enum ToolPackFeatureGroup { BrowserWeb, ComputerUse, ImageAnalysis, + ImageGeneration, MiniApp, Pages, Creation, @@ -31,6 +32,7 @@ impl ToolPackFeatureGroup { Self::BrowserWeb => "browser-web", Self::ComputerUse => "computer-use", Self::ImageAnalysis => "image-analysis", + Self::ImageGeneration => "image-generation", Self::MiniApp => "miniapp", Self::Pages => "pages", Self::Creation => "creation", @@ -47,6 +49,7 @@ pub const ALL_FEATURE_GROUPS: &[ToolPackFeatureGroup] = &[ ToolPackFeatureGroup::BrowserWeb, ToolPackFeatureGroup::ComputerUse, ToolPackFeatureGroup::ImageAnalysis, + ToolPackFeatureGroup::ImageGeneration, ToolPackFeatureGroup::MiniApp, ToolPackFeatureGroup::Pages, ToolPackFeatureGroup::Creation, @@ -75,6 +78,10 @@ pub fn enabled_feature_groups() -> Vec { cfg!(feature = "image-analysis"), ToolPackFeatureGroup::ImageAnalysis, ), + ( + cfg!(feature = "image-generation"), + ToolPackFeatureGroup::ImageGeneration, + ), (cfg!(feature = "miniapp"), ToolPackFeatureGroup::MiniApp), (cfg!(feature = "pages"), ToolPackFeatureGroup::Pages), (cfg!(feature = "creation"), ToolPackFeatureGroup::Creation), @@ -101,9 +108,8 @@ pub fn tool_feature_group(tool_name: &str) -> Option { } "WebSearch" | "WebFetch" | "ControlHub" => Some(ToolPackFeatureGroup::BrowserWeb), "ComputerUse" => Some(ToolPackFeatureGroup::ComputerUse), - "view_image" | "analyze_image" | "generate_image" => { - Some(ToolPackFeatureGroup::ImageAnalysis) - } + "view_image" | "analyze_image" => Some(ToolPackFeatureGroup::ImageAnalysis), + "generate_image" => Some(ToolPackFeatureGroup::ImageGeneration), "GenerativeUI" | "InitMiniApp" | "FinalizeMiniApp" | "PublishMiniApp" | "PublishAppearance" | "Playbook" => Some(ToolPackFeatureGroup::MiniApp), "PageDeploy" | "PagePublish" => Some(ToolPackFeatureGroup::Pages), @@ -157,6 +163,7 @@ impl ToolProviderGroupPlan { const CORE_BASIC_FEATURE_GROUPS: &[ToolPackFeatureGroup] = &[ ToolPackFeatureGroup::Basic, ToolPackFeatureGroup::ImageAnalysis, + ToolPackFeatureGroup::ImageGeneration, ]; const CORE_AGENT_FEATURE_GROUPS: &[ToolPackFeatureGroup] = &[ToolPackFeatureGroup::AgentControl]; const CORE_REVIEW_FEATURE_GROUPS: &[ToolPackFeatureGroup] = &[ToolPackFeatureGroup::AgentControl]; @@ -357,6 +364,7 @@ mod tests { "browser-web", "computer-use", "image-analysis", + "image-generation", "miniapp", "pages", "creation", @@ -394,6 +402,10 @@ mod tests { groups.contains(&ToolPackFeatureGroup::ImageAnalysis), cfg!(feature = "image-analysis") ); + assert_eq!( + groups.contains(&ToolPackFeatureGroup::ImageGeneration), + cfg!(feature = "image-generation") + ); assert_eq!( groups.contains(&ToolPackFeatureGroup::MiniApp), cfg!(feature = "miniapp") @@ -438,6 +450,20 @@ mod tests { } } + #[test] + fn generation_and_inspection_have_independent_feature_owners() { + assert_eq!( + tool_feature_group("generate_image"), + Some(ToolPackFeatureGroup::ImageGeneration) + ); + for tool_name in ["view_image", "analyze_image"] { + assert_eq!( + tool_feature_group(tool_name), + Some(ToolPackFeatureGroup::ImageAnalysis) + ); + } + } + #[test] fn every_provider_declares_exactly_its_tool_owner_groups() { for provider in product_tool_provider_group_plan() { @@ -479,6 +505,10 @@ mod tests { assert_eq!(ToolPackFeatureGroup::BrowserWeb.id(), "browser-web"); assert_eq!(ToolPackFeatureGroup::ComputerUse.id(), "computer-use"); assert_eq!(ToolPackFeatureGroup::ImageAnalysis.id(), "image-analysis"); + assert_eq!( + ToolPackFeatureGroup::ImageGeneration.id(), + "image-generation" + ); assert_eq!(ToolPackFeatureGroup::MiniApp.id(), "miniapp"); assert_eq!(ToolPackFeatureGroup::Creation.id(), "creation"); assert_eq!(ToolPackFeatureGroup::Canvas.id(), "canvas"); @@ -525,6 +555,7 @@ mod tests { "Read", "view_image", "analyze_image", + "generate_image", "Glob", "Grep", "Write", @@ -606,7 +637,10 @@ mod tests { assert_eq!( feature_groups, vec![ - ("core.basic", vec!["basic", "image-analysis"]), + ( + "core.basic", + vec!["basic", "image-analysis", "image-generation"] + ), ("core.agent", vec!["agent-control"]), ("core.session", vec!["agent-control"]), ("core.git", vec!["git"]), diff --git a/src/crates/interfaces/acp/Cargo.toml b/src/crates/interfaces/acp/Cargo.toml index 4315583085..5449422ba9 100644 --- a/src/crates/interfaces/acp/Cargo.toml +++ b/src/crates/interfaces/acp/Cargo.toml @@ -35,6 +35,7 @@ server = [ "openbitfun-core/tools-browser-web", "openbitfun-core/tools-computer-use", "openbitfun-core/tools-image-analysis", + "openbitfun-core/tools-image-generation", "openbitfun-core/tools-agent-control", ] diff --git a/src/crates/services/services-core/AGENTS.md b/src/crates/services/services-core/AGENTS.md index dbd9fd7e3f..9c3f91c5ce 100644 --- a/src/crates/services/services-core/AGENTS.md +++ b/src/crates/services/services-core/AGENTS.md @@ -87,6 +87,7 @@ target. Representative stable entry points are: ```bash cargo check -p openbitfun-services-core --no-default-features +cargo test --locked -p openbitfun-services-core --no-default-features --features system-proxy --lib system_proxy::tests cargo test -p openbitfun-services-core --no-default-features --features pet-packages --lib pet_packages cargo test -p openbitfun-services-core --no-default-features --features asar --lib asar cargo test -p openbitfun-services-core --no-default-features --features process-runtime --lib system::info::tests diff --git a/src/shared/interactive-capabilities/catalog.json b/src/shared/interactive-capabilities/catalog.json index 0945556e76..36fe42a4fc 100644 --- a/src/shared/interactive-capabilities/catalog.json +++ b/src/shared/interactive-capabilities/catalog.json @@ -4,6 +4,18 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "searchAcceptance": [ + { + "id": "native-image-generation", + "query": "generate_image", + "expectedFirstCapabilityId": "feature.ai-assistant", + "expectedCapabilityIds": [ + "feature.ai-assistant" + ], + "expectedItem": { + "capabilityId": "feature.ai-assistant", + "itemId": "image-generation" + } + }, { "id": "companion-pet-mixed", "query": "宠物 pet mascot 桌面宠物", @@ -416,6 +428,31 @@ "command:frontend_update_candidate_failed" ] }, + { + "id": "image-generation", + "titleZh": "通过已配置的默认图像模型生成图片", + "titleEn": "Generate images with the configured default image model", + "control": { + "kind": "delegate", + "tools": [ + "generate_image" + ], + "workflowZh": [ + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务" + ], + "workflowEn": [ + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services" + ] + }, + "evidence": [ + "source:src/crates/assembly/core/src/agentic/tools/implementations/generate_image_tool.rs#generate_image", + "source:src/crates/assembly/core/builtin_skills/generate-image/SKILL.md#generate_image" + ] + }, { "id": "turn-control", "titleZh": "开始、引导、打断、取消或恢复一次智能体运行", @@ -6934,17 +6971,18 @@ }, { "id": "connection-test", - "titleZh": "保存前测试供应商或模型连接并刷新运行时客户端", - "titleEn": "Test provider or model connectivity before saving and refresh the runtime client", + "titleZh": "测试模型连接或图像生成，并刷新运行时客户端", + "titleEn": "Test model connections or image generation and refresh the runtime client", "control": { "kind": "open", "reasonCode": "unstructuredInteraction", - "reasonZh": "“保存前测试供应商或模型连接并刷新运行时客户端”由多个实时状态相关步骤组成，目前没有一个能确定完成整个流程的单一结构化 Command；智能体会打开精确入口，并把后续交互保留在用户可见界面。", - "reasonEn": "“Test provider or model connectivity before saving and refresh the runtime client” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user." + "reasonZh": "“测试模型连接或图像生成，并刷新运行时客户端”需要用户在配置表单中选择服务并主动发起；图像服务测试会实际生成图片并可能消耗额度，智能体只能打开对应设置入口。", + "reasonEn": "“Test model connections or image generation and refresh the runtime client” requires the user to select a configuration and start the test from its form; image-service tests generate an actual image and may consume quota, so the Agent can only open the settings entry." }, "evidence": [ "command:test_ai_connection", "command:test_ai_config_connection", + "command:test_image_generation_service", "command:refresh_model_client" ] }, diff --git a/src/web-ui/src/app/global-search/generated/interactive-capabilities.json b/src/web-ui/src/app/global-search/generated/interactive-capabilities.json index 5fde780727..096cc2e233 100644 --- a/src/web-ui/src/app/global-search/generated/interactive-capabilities.json +++ b/src/web-ui/src/app/global-search/generated/interactive-capabilities.json @@ -4,9 +4,21 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ + { + "id": "native-image-generation", + "query": "generate_image", + "expectedFirstCapabilityId": "feature.ai-assistant", + "expectedCapabilityIds": [ + "feature.ai-assistant" + ], + "expectedItem": { + "capabilityId": "feature.ai-assistant", + "itemId": "image-generation" + } + }, { "id": "companion-pet-mixed", "query": "宠物 pet mascot 桌面宠物", @@ -147,10 +159,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -202,6 +214,7 @@ "session-lifecycle", "model-mode-permissions", "creative-frontend-authoring", + "image-generation", "turn-control", "permission-mailbox", "history-search", @@ -443,6 +456,51 @@ "sceneId": "session" } }, + { + "id": "feature.ai-assistant:delegate:image-generation", + "capabilityId": "feature.ai-assistant", + "itemIds": [ + "image-generation" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "generate_image" + ], + "presentationTarget": { + "kind": "scene", + "sceneId": "session" + } + }, { "id": "feature.ai-assistant:open:turn-control", "capabilityId": "feature.ai-assistant", @@ -19823,6 +19881,27 @@ ] } }, + { + "id": "image-generation", + "titleZh": "通过已配置的默认图像模型生成图片", + "titleEn": "Generate images with the configured default image model", + "control": { + "kind": "delegate", + "tools": [ + "generate_image" + ], + "workflowZh": [ + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务" + ], + "workflowEn": [ + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services" + ] + } + }, { "id": "turn-control", "titleZh": "开始、引导、打断、取消或恢复一次智能体运行", @@ -20085,6 +20164,8 @@ "Choose the model, work mode, memory mode, and permission mode per session", "让创造模式扩展客户端界面与运行时能力", "Extend client UI and runtime capabilities in Creative mode", + "通过已配置的默认图像模型生成图片", + "Generate images with the configured default image model", "开始、引导、打断、取消或恢复一次智能体运行", "Start, steer, interrupt, cancel, or recover an agent turn", "查看并逐个或批量回答等待中的权限请求", @@ -20125,6 +20206,13 @@ "Call FrontendWorkbench prepare on the local Desktop in Creative mode, read the packaged API, and edit only draft CSS, JavaScript and owned assets without source or build tools", "Apply with the original draft ID; the host starts the immutable review window's 15-second countdown only after the real shell and customization activate, and apply waits for a final confirmed or rolled_back outcome", "Compose commands, state and events into reusable capabilities; use FrontendWorkbench inspect to discover schemas and diagnostics, then invoke to verify execution", + "generate_image", + "使用 generate_image 按用户描述生成图片，默认省略 service_id 以使用已配置的默认图像模型", + "按需指定数量和尺寸，通过返回的文件路径交付图片；远程工作区文件通过当前工作区数据面读写", + "缺少配置、认证失败或额度不足时明确报告，不自动切换到其他服务", + "Use generate_image to create images from the user's description; omit service_id to use the configured default image model", + "Specify count and size when needed and deliver the returned file paths; access remote workspace files through the active workspace data plane", + "Report missing configuration, authentication failures, or quota limits explicitly without automatically switching services", "ListWorkspaces", "SessionControl", "SessionHistory", @@ -27559,13 +27647,13 @@ }, { "id": "connection-test", - "titleZh": "保存前测试供应商或模型连接并刷新运行时客户端", - "titleEn": "Test provider or model connectivity before saving and refresh the runtime client", + "titleZh": "测试模型连接或图像生成，并刷新运行时客户端", + "titleEn": "Test model connections or image generation and refresh the runtime client", "control": { "kind": "open", "reasonCode": "unstructuredInteraction", - "reasonZh": "“保存前测试供应商或模型连接并刷新运行时客户端”由多个实时状态相关步骤组成，目前没有一个能确定完成整个流程的单一结构化 Command；智能体会打开精确入口，并把后续交互保留在用户可见界面。", - "reasonEn": "“Test provider or model connectivity before saving and refresh the runtime client” spans multiple live-state-dependent steps and currently has no single structured Command that can deterministically complete the whole workflow; the Agent opens the exact entry and keeps the remaining interaction visible to the user." + "reasonZh": "“测试模型连接或图像生成，并刷新运行时客户端”需要用户在配置表单中选择服务并主动发起；图像服务测试会实际生成图片并可能消耗额度，智能体只能打开对应设置入口。", + "reasonEn": "“Test model connections or image generation and refresh the runtime client” requires the user to select a configuration and start the test from its form; image-service tests generate an actual image and may consume quota, so the Agent can only open the settings entry." } }, { @@ -27679,8 +27767,8 @@ "Inspect models.dev catalog status, refresh it manually, and reveal its local cache", "为供应商添加自定义 Header、请求体字段与 SSL 行为", "Add custom headers, request-body fields, and SSL behavior per provider", - "保存前测试供应商或模型连接并刷新运行时客户端", - "Test provider or model connectivity before saving and refresh the runtime client", + "测试模型连接或图像生成，并刷新运行时客户端", + "Test model connections or image generation and refresh the runtime client", "管理模型池、路由策略与会话模型选择", "Manage model pools, routing strategies, and session model selection", "打开模型设置", diff --git a/src/web-ui/src/app/global-search/providers/interactiveCapabilitySearchProvider.test.ts b/src/web-ui/src/app/global-search/providers/interactiveCapabilitySearchProvider.test.ts index 5e9bb8aef1..d453094995 100644 --- a/src/web-ui/src/app/global-search/providers/interactiveCapabilitySearchProvider.test.ts +++ b/src/web-ui/src/app/global-search/providers/interactiveCapabilitySearchProvider.test.ts @@ -52,6 +52,17 @@ describe('interactiveCapabilitySearchProvider', () => { } }); + it.each(['en-US', 'zh-CN', 'zh-TW'] as const)('discovers image generation with a localized %s label', async locale => { + const result = await interactiveCapabilitySearchProvider.search( + request('generate_image', locale), + new AbortController().signal, + ); + const imageGeneration = result.items.find(item => item.target.kind === 'capability' + && item.target.capabilityId === 'feature.ai-assistant'); + expect(imageGeneration?.target).toMatchObject({ kind: 'capability', itemId: 'image-generation' }); + expect(imageGeneration?.subtitle).toBe(translate(locale, 'search.capabilities.feature.ai-assistant.items.image-generation')); + }); + it('restores accelerated search in the settings catalog', () => { const capability = INTERACTIVE_CAPABILITY_CATALOG.capabilities.find( ({ id }) => id === 'setting.workspace.session', diff --git a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx index 6750af70e3..af016628ec 100644 --- a/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx +++ b/src/web-ui/src/app/scenes/settings/pages/ai/ModelSettingsPage.loading.test.tsx @@ -49,7 +49,7 @@ describe('ModelSettingsPage loading and provider actions', () => { )]; beforeAll(async () => { - await Promise.all((['settings', 'settings/models', 'settings/default-model', 'components'] as const).map( + await Promise.all((['settings', 'settings/models', 'settings/default-model', 'settings/media', 'components'] as const).map( namespace => i18nService.loadNamespace(namespace), )); }); @@ -59,6 +59,7 @@ describe('ModelSettingsPage loading and provider actions', () => { reads = new Map([ 'ai.models', 'ai.default_models', 'ai.proxy', 'ai.stream_idle_timeout_secs', 'ai.stream_ttft_timeout_secs', + 'media', ].map(path => [path, deferred()])); vi.spyOn(configManager, 'getOptionalConfig').mockImplementation( (path: string) => reads.get(path)!.promise as Promise, @@ -158,6 +159,7 @@ describe('ModelSettingsPage loading and provider actions', () => { it('renders a warm model pool immediately without starting a read that could overwrite edits', () => { const values: Record = { + media: { image_services: [] }, 'ai.models': [model], 'ai.default_models': { primary: model.id }, 'ai.proxy': { enabled: false, url: '', username: '', password: '' }, 'ai.stream_idle_timeout_secs': 120, 'ai.stream_ttft_timeout_secs': 60, @@ -341,7 +343,11 @@ describe('ModelSettingsPage loading and provider actions', () => { for (const key of ['acquisition.noneConfigured', 'acquisition.noneImported', 'pool.empty']) { expect(container.textContent).not.toContain(copy(key)); } - expect(selectors()).toHaveLength(4); + const defaultLabel = (key: string) => String(i18nService.getI18nInstance().t(key, { ns: 'settings/default-model' })); + expect(selectors().map(element => element.getAttribute('aria-label'))).toEqual([ + 'core.primary.label', 'core.fast.label', 'optional.capabilities.image_understanding.label', + 'optional.capabilities.image_generation.label', 'optional.capabilities.speech_recognition.label', + ].map(defaultLabel)); expect(selectors().every(element => element.disabled)).toBe(true); expect(button('actions.addModel').disabled).toBe(true); expect(button('acquisition.import').disabled).toBe(false); @@ -355,6 +361,17 @@ describe('ModelSettingsPage loading and provider actions', () => { expect(selectors()[0].disabled).toBe(false); expect(selectors()[0].textContent).toContain(model.model_name); expect(button('actions.addModel').disabled).toBe(false); + const imageSelector = container.querySelector('[data-testid="image-service-default-select"] button[role="combobox"]')!; + expect(imageSelector.disabled).toBe(true); + const image = { + id: 'loading-image-service', name: 'Image service', dialect: 'openai_images', + base_url: 'https://images.example/v1', model_name: 'image-model', api_key: 'fixture-key', + enabled: true, params: { n: 1 }, + }; + await act(async () => { reads.get('media')!.resolve({ image_services: [image], default_image_service: image.id }); }); + expect(imageSelector.disabled).toBe(false); + expect(imageSelector.textContent).toContain(image.name); + expect(selectors()[0].textContent).toContain(model.model_name); }); it('isolates a failed model read, keeps network settings usable, and recovers through the section retry', async () => { diff --git a/src/web-ui/src/flow_chat/components/FlowTextBlock.tsx b/src/web-ui/src/flow_chat/components/FlowTextBlock.tsx index 3ec0f83284..2c86d71ff9 100644 --- a/src/web-ui/src/flow_chat/components/FlowTextBlock.tsx +++ b/src/web-ui/src/flow_chat/components/FlowTextBlock.tsx @@ -166,6 +166,7 @@ export const FlowTextBlock = React.memo(({ ; diff --git a/src/web-ui/src/infrastructure/api/generated/productControl.ts b/src/web-ui/src/infrastructure/api/generated/productControl.ts index 13b55d2052..b61576a637 100644 --- a/src/web-ui/src/infrastructure/api/generated/productControl.ts +++ b/src/web-ui/src/infrastructure/api/generated/productControl.ts @@ -1,5 +1,5 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. -export const PRODUCT_CONTROL_GRAPH_DIGEST = "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5" as const; +export const PRODUCT_CONTROL_GRAPH_DIGEST = "64bbd3571c458cdd0b67ba9337820e88f9b52d5edf4aefc8320874643dce7a77" as const; export type ProductControlCapabilityId = "feature.ai-assistant" | "feature.agents" | "feature.personal-assistants" | "feature.projects" | "feature.files-editor" | "feature.terminal" | "feature.git" | "feature.code-review" | "feature.browser" | "feature.computer-use" | "feature.skills" | "feature.miniapps" | "feature.canvas" | "feature.tasks-automation" | "feature.insights" | "feature.ecosystem-compatibility" | "feature.remote-workspaces" | "feature.remote-connect" | "feature.detached-dispatch" | "feature.pages" | "feature.voice-input" | "feature.desktop-pet" | "feature.desktop-cards" | "setting.application.general" | "setting.application.appearance" | "setting.application.pet" | "setting.application.input" | "setting.application.shortcuts" | "setting.application.development" | "setting.ai.models" | "setting.ai.memory" | "setting.workspace.session" | "setting.workspace.worktrees" | "setting.tools.execution" | "setting.application.terminal" | "setting.tools.desktop-control" | "setting.tools.browser-control" | "setting.tools.automation" | "setting.tools.web-search" | "setting.tools.mcp" | "setting.tools.acp" | "setting.data.usage" | "setting.data.archived" | "setting.data.diagnostics"; diff --git a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts index 380183ca85..b3d14369b4 100644 --- a/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts +++ b/src/web-ui/src/infrastructure/api/generated/remoteSurface.ts @@ -1,6 +1,6 @@ // Generated by scripts/generate-interactive-capabilities.mjs; do not edit. // Source: openbitfun_product_domains::remote_surface (Product Operation Registry). -export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:eb8901b156df1dd3" as const; +export const REMOTE_SURFACE_REGISTRY_DIGEST = "fnv1a64:d64185f2013646ad" as const; /** * Registered Tauri commands the Peer Device controller keeps on the controller diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationEditor.tsx b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationEditor.tsx index 94f63b130b..9c5f282a4b 100644 --- a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationEditor.tsx +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationEditor.tsx @@ -1,12 +1,12 @@ import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { Button, Combobox, Dialog, DialogBody, DialogClose, DialogFooter, DialogHeader, DialogHeading, DialogTitle, Disclosure, Input, NumberInput, Switch } from '@openbitfun/ui'; +import { Button, Combobox, Dialog, DialogBody, DialogClose, DialogFooter, DialogHeader, DialogHeading, DialogTitle, Input } from '@openbitfun/ui'; import { ConfigPageRow } from '../common'; import { mediaAPI, type ImageServiceTestOutcome } from '@/infrastructure/api/service-api/MediaApi'; import { useI18n } from '@/infrastructure/i18n'; import { getActiveSurfaceScope } from '@/infrastructure/peer-device/deviceSurface'; import { useSettingsDraft } from '../../settingsDraftRegistry'; -import type { AuthConfig, ImageDialect, ImageGenerationParams, ImageServiceConfig } from '../../types'; -import { findImageDialectPreset, IMAGE_DIALECT_PRESETS, resolveImageEndpoint, withImageServiceAuth } from './imageDialects'; +import type { AuthConfig, ImageDialect, ImageServiceConfig } from '../../types'; +import { findImageDialectPreset, IMAGE_DIALECT_PRESETS, withImageServiceAuth } from './imageDialects'; import { duplicateImageService } from './imageModels'; export interface ReusableImageCredential { @@ -52,7 +52,6 @@ export default function ImageGenerationEditor({ open, initial, credentials, serv version.current += 1; setOutcome(undefined); setTesting(false); setDraft(current => ({ ...current, ...changes })); }, []); - const patchParams = (changes: Partial) => patch({ params: { ...draft.params, ...changes } }); const subscription = draft.auth?.type === 'subscription' || draft.auth?.type === 'subscription_pool'; const preset = findImageDialectPreset(draft.dialect); const duplicate = duplicateImageService(services, draft); @@ -136,49 +135,14 @@ export default function ImageGenerationEditor({ open, initial, credentials, serv patch({ model_name: value })} /> - patch({ base_url: value })} /> + patch(draft.request_url !== undefined ? { request_url: value } : { base_url: value })} /> patch({ api_key: value })} /> } - -
- patch({ name: value })} disabled={saving || testing} /> - {subscription && - patch({ model_name: value })} disabled={saving || testing} placeholder={t('wizard.fields.modelName.subscriptionPlaceholder')} /> - } - - ({ label: value, value }))]} - disabled={saving || testing} onCreateValue={value => value.trim() || undefined} - onValueChange={value => patchParams({ size: String(value) || undefined })} /> - - {preset.maxImagesPerRequest > 1 && - patchParams({ n })} disabled={saving || testing} /> - } - {preset.supports.quality && - ({ label: value, value }))} - onValueChange={value => patchParams({ quality: String(value) })} disabled={saving || testing} /> - } - {preset.supports.background && - ({ label: value, value }))} - onValueChange={value => patchParams({ background: String(value) })} disabled={saving || testing} /> - } - {preset.supports.negativePrompt && - patchParams({ negative_prompt })} disabled={saving || testing} /> - } - {preset.supports.watermark && - patchParams({ watermark: event.target.checked })} disabled={saving || testing} /> - } - {preset.supports.promptExtend && - patchParams({ prompt_extend: event.target.checked })} disabled={saving || testing} /> - } - {!subscription && - patch({ request_url })} disabled={saving || testing} /> - } - {!subscription &&

{resolveImageEndpoint(draft.base_url, draft.request_url, draft.dialect)}

} -
- + {outcome &&

{outcome.success ? t('wizard.test.success', { time: formatNumber(outcome.elapsed_ms) }) : t(`wizard.errors.${outcome.error_kind || 'provider'}`)}

{outcome.message &&

{outcome.message}

} diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss index 3be09a17d9..1f0e0d8a38 100644 --- a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSection.scss @@ -1,5 +1,4 @@ -.openbitfun-image-settings__editor, -.openbitfun-image-settings__advanced { +.openbitfun-image-settings__editor { display: flex; flex-direction: column; gap: var(--openbitfun-space-3); @@ -13,7 +12,6 @@ margin: 0; color: var(--openbitfun-color-content-secondary); } -.openbitfun-image-settings__endpoint, .openbitfun-image-settings__test-detail { margin: 0; overflow-wrap: anywhere; diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx index 4b08b07173..39a67947f7 100644 --- a/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx +++ b/src/web-ui/src/infrastructure/config/components/image-generation/ImageGenerationSettings.test.tsx @@ -103,9 +103,11 @@ describe('image models in model settings', () => { await render(); await click(host.querySelector('[data-testid="image-service-add"]')!); await choose('image-service-source', 'My subscription'); expect(document.querySelector('[role="dialog"] input[type="password"]')).toBeNull(); - expect(button('advanced')?.getAttribute('aria-expanded')).toBe('false'); + expect(document.querySelector('[data-testid="image-service-advanced"]')).toBeNull(); + expect(document.querySelector('[role="dialog"] input')).toBeNull(); await click(button('actions.save')); expect(media.image_services.at(-1)).toMatchObject({ name: 'My subscription', dialect: 'chatgpt_codex', model_name: '', api_key: '' }); + expect(media.image_services.at(-1)?.params.size).toBeUndefined(); expect(document.querySelector('[role="dialog"]:not([aria-hidden="true"])')).toBeNull(); expect(mocks.test).not.toHaveBeenCalled(); }); @@ -169,6 +171,30 @@ describe('image models in model settings', () => { expect(media.default_image_service).toBe('legacy'); }); + it('shows the effective legacy request address without exposing generation defaults', async () => { + await render(); + await click([...host.querySelectorAll('button')].find(element => element.textContent?.includes('Saved image'))!); + const dialog = document.querySelector('[role="dialog"]')!; + expect([...dialog.querySelectorAll('input')].map(input => input.value)).toContain(legacy.image_services[0].request_url); + for (const field of ['name', 'size', 'n', 'quality', 'background', 'negativePrompt', 'watermark', 'promptExtend', 'requestUrl']) { + expect(dialog.textContent).not.toContain(`wizard.fields.${field}.label`); + } + const address = [...dialog.querySelectorAll('input')].find(input => input.value === legacy.image_services[0].request_url)!; + const updatedAddress = 'https://images.example/custom/generate'; + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')!.set!.call(address, updatedAddress); + address.dispatchEvent(new Event('input', { bubbles: true })); + }); + await click(button('actions.save')); + expect(media.image_services[0]).toMatchObject({ + request_url: updatedAddress, + base_url: legacy.image_services[0].base_url, + params: legacy.image_services[0].params, + custom_headers: legacy.image_services[0].custom_headers, + }); + expect(mocks.test).not.toHaveBeenCalled(); + }); + it('keeps an image-only configuration selectable when there are no chat models', async () => { await render([]); const control = host.querySelector('[data-testid="image-service-default-select"] button')!; diff --git a/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts index 1f467e5a06..245eef0146 100644 --- a/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts +++ b/src/web-ui/src/infrastructure/config/components/image-generation/imageDialects.ts @@ -110,7 +110,7 @@ export function withImageServiceAuth(service: ImageServiceConfig, auth: AuthConf base_url: preset.defaultBaseUrl, request_url: undefined, model_name: preset.defaultModel, - params: { ...service.params, size: preset.sizes[0], n: 1 }, + params: { ...service.params, size: undefined, n: 1 }, }; } diff --git a/src/web-ui/src/infrastructure/markdown/Markdown.scss b/src/web-ui/src/infrastructure/markdown/Markdown.scss index 9cde68110f..aa5d73058f 100644 --- a/src/web-ui/src/infrastructure/markdown/Markdown.scss +++ b/src/web-ui/src/infrastructure/markdown/Markdown.scss @@ -418,6 +418,55 @@ cursor: zoom-in; } +/* Conversation previews stay compact; the lightbox retains the original bytes. */ +.markdown-renderer--thumbnails img { + max-width: min(100%, 18rem); + max-height: 14rem; + object-fit: contain; +} + +.markdown-renderer .markdown-image-gallery { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(100%, 9rem), 1fr)); + gap: var(--openbitfun-space-2); + width: min(100%, 32rem); + margin-block: var(--openbitfun-space-2); +} + +.markdown-renderer .markdown-image-gallery--single { + width: min(100%, 18rem); + grid-template-columns: minmax(0, 1fr); +} + +.markdown-renderer .markdown-image-gallery--paired { + width: min(100%, 26rem); +} + +.markdown-renderer .markdown-image-tile { + display: flex; + align-items: center; + justify-content: center; + min-width: 0; + aspect-ratio: 4 / 3; + overflow: hidden; + border-radius: var(--openbitfun-control-flow-chat-card-radius); + background: var(--openbitfun-color-surface-subtle); +} + +.markdown-renderer .markdown-image-tile > img { + width: 100%; + height: 100%; + max-width: 100%; + max-height: 100%; + margin: 0; + object-fit: contain; +} + +.markdown-renderer .markdown-image-thumbnail:focus-visible { + outline: var(--openbitfun-border-width-default) solid var(--openbitfun-color-border-default); + outline-offset: calc(-1 * var(--openbitfun-space-1)); +} + /* A linked image belongs to its link, not to the preview affordance. */ .markdown-renderer :is(a, button):has(> img) > img.markdown-image--previewable { cursor: pointer; diff --git a/src/web-ui/src/infrastructure/markdown/MarkdownMathRenderer.tsx b/src/web-ui/src/infrastructure/markdown/MarkdownMathRenderer.tsx index c5d6a7203a..e9c4b35d67 100644 --- a/src/web-ui/src/infrastructure/markdown/MarkdownMathRenderer.tsx +++ b/src/web-ui/src/infrastructure/markdown/MarkdownMathRenderer.tsx @@ -13,6 +13,7 @@ import type { Pluggable } from 'unified'; import 'katex/dist/katex.min.css'; import { rehypeSourceRange, type MarkdownSourceRange } from './rehypeSourceRange'; import { rehypeWindowsDrivePaths } from './rehypeWindowsDrivePaths'; +import { rehypeImageThumbnails } from './rehypeImageThumbnails'; interface MarkdownMathRendererProps { markdownContent: string; @@ -23,6 +24,7 @@ interface MarkdownMathRendererProps { urlTransform: (value: string) => string; sourceRange?: MarkdownSourceRange; inline?: boolean; + imageThumbnails?: boolean; } export const MarkdownMathRenderer: React.FC = ({ @@ -34,11 +36,12 @@ export const MarkdownMathRenderer: React.FC = ({ urlTransform, sourceRange, inline = false, + imageThumbnails = false, }) => { const content = ( diff --git a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.test.tsx b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.test.tsx index b9994c192e..2090d6a707 100644 --- a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.test.tsx +++ b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.test.tsx @@ -1152,6 +1152,65 @@ Second paragraph. ); }); + it('groups conversation images across paragraphs and line breaks while preserving text and linked badges', async () => { + const content = 'Before\n\n![Landscape](data:image/png;base64,YQ==)\n\n![Portrait](data:image/png;base64,Yg==)\n![Third](data:image/png;base64,Yw==)\n\nAfter\n\n[![Badge](https://example.com/badge.png)](https://example.com)'; + await act(async () => root.render()); + const galleries = container.querySelectorAll('.markdown-image-gallery'); + expect(galleries).toHaveLength(1); + expect(galleries[0].querySelectorAll('.markdown-image-tile')).toHaveLength(3); + expect(galleries[0].classList.contains('markdown-image-gallery--single')).toBe(false); + expect([...container.querySelectorAll('.markdown-paragraph')].map(node => node.textContent)).toEqual(['Before', 'After', '']); + expect(container.querySelector('a img[alt="Badge"]')).not.toBeNull(); + expect(container.querySelector('a img[alt="Badge"]')?.getAttribute('role')).toBeNull(); + expect(container.querySelector('.markdown-renderer--thumbnails')).not.toBeNull(); + }); + + it('keeps document images in their original paragraphs unless thumbnails are requested', async () => { + await act(async () => root.render()); + expect(container.querySelector('.markdown-image-gallery')).toBeNull(); + expect(container.querySelector('.markdown-paragraph > img')).not.toBeNull(); + }); + + it('opens the selected original image by keyboard and retains resolved images as the gallery streams', async () => { + const content = '![First](data:image/png;base64,YQ==)'; + await act(async () => root.render()); + const first = container.querySelector('img'); + expect(container.querySelector('.markdown-image-gallery--single')).not.toBeNull(); + await act(async () => root.render()); + expect(container.querySelector('img')).toBe(first); + const second = container.querySelector('img[alt="Second"]')!; + expect(second.getAttribute('tabindex')).toBe('0'); + await act(async () => second.dispatchEvent(new KeyboardEvent('keydown', { key: 'Enter', bubbles: true }))); + expect(document.querySelector('.image-lightbox img')?.getAttribute('src')).toBe('data:image/png;base64,Yg=='); + await act(async () => document.querySelector('.image-lightbox-close')?.click()); + expect(document.querySelector('.image-lightbox')).toBeNull(); + }); + + it('previews dispatch gallery bytes from the session reader without accessing the controller filesystem', async () => { + const read = vi.fn().mockResolvedValue('data:image/png;base64,dGFyZ2V0'); + await act(async () => root.render()); + expect(read).toHaveBeenCalledWith('/target/image.png'); + expect(mocks.readFileContent).not.toHaveBeenCalled(); + const image = container.querySelector('.markdown-image-tile img')!; + await act(async () => image.dispatchEvent(new KeyboardEvent('keydown', { key: ' ', bubbles: true }))); + expect(document.querySelector('.image-lightbox img')?.getAttribute('src')).toBe('data:image/png;base64,dGFyZ2V0'); + await act(async () => activateSurface('peer:other-gallery-host')); + expect(document.querySelector('.image-lightbox')).toBeNull(); + }); + + it('keeps failed session images retryable inside a thumbnail tile', async () => { + const read = vi.fn().mockRejectedValueOnce(new Error('target offline')).mockResolvedValueOnce('data:image/png;base64,b2s='); + await act(async () => root.render()); + const tile = container.querySelector('.markdown-image-tile')!; + expect(tile.querySelector('img')).toBeNull(); + const retry = [...tile.querySelectorAll('button')].find(button => button.textContent?.includes('common:retry'))!; + await act(async () => retry.click()); + expect(read).toHaveBeenLastCalledWith('/target/retry.png', true); + expect(tile.querySelector('img')?.getAttribute('src')).toBe('data:image/png;base64,b2s='); + }); + it('previews the resolved bytes of a markdown image and closes on the scrim or the close button', async () => { await act(async () => { root.render( diff --git a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx index 445b63e1ea..4c15510d25 100644 --- a/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx +++ b/src/web-ui/src/infrastructure/markdown/MarkdownRenderer.tsx @@ -13,6 +13,7 @@ import remarkGfm from 'remark-gfm'; import { remarkCachedParse } from './markdownParseCache'; import { remarkAutolinkBoundaries } from './remarkAutolinkBoundaries'; import { remarkStreamingTableLinks } from './remarkStreamingTableLinks'; +import { rehypeImageThumbnails } from './rehypeImageThumbnails'; import rehypeRaw from 'rehype-raw'; import rehypeSanitize, { defaultSchema } from 'rehype-sanitize'; import { visit } from 'unist-util-visit'; @@ -572,6 +573,7 @@ const ScopedMarkdownImage: React.FC { const fileAccess = useResourceFileAccess(); @@ -671,6 +673,7 @@ const ScopedMarkdownImage: React.FC { + if (previewable && !event.currentTarget.closest('a, button') && ['Enter', ' '].includes(event.key)) { + event.preventDefault(); + event.stopPropagation(); + onPreview?.(resolvedSrc, typeof alt === 'string' && alt ? alt : undefined); + } else onKeyDown?.(event); + }} onClick={(event) => { // An image owned by a link or a file link keeps that owner's behavior. if (!previewable || event.currentTarget.closest('a, button')) { @@ -878,6 +890,8 @@ const CopyButton: React.FC<{ code: string }> = ({ code }) => { export interface MarkdownRendererProps { content: string; + /** Compact conversation images; document renderers retain their original layout. */ + imageThumbnails?: boolean; /** Display a region while resolving Markdown references against all content. */ sourceRange?: MarkdownSourceRange; /** Owning workspace ID; content opened from links is routed by it. `basePath` is only the IO root. */ @@ -909,6 +923,7 @@ const MarkdownSurface = React.memo; + read={onImageReadRef.current} download={onFileDownloadRef.current} onPreview={onImagePreviewRef.current} + thumbnail={props.className?.split(' ').includes('markdown-image-thumbnail')} />; } // Dispatch observers have no local filesystem ownership. Do not mount // MarkdownImage here: even its initial state can reuse controller bytes @@ -1725,7 +1741,7 @@ const MarkdownSurface = React.memo ({ fileAccess, surfaceScope, currentWorkspacePath, workspaceId, basePath, remoteConnectionId, remoteSshHost, onImageRead, onFileDownload, @@ -1762,7 +1778,7 @@ const MarkdownSurface = React.memo @@ -1806,6 +1822,7 @@ const MarkdownSurface = React.memo ) : basicMarkdownRenderer} diff --git a/src/web-ui/src/infrastructure/markdown/SessionMarkdownImage.tsx b/src/web-ui/src/infrastructure/markdown/SessionMarkdownImage.tsx index 596594ea38..0716e6db0b 100644 --- a/src/web-ui/src/infrastructure/markdown/SessionMarkdownImage.tsx +++ b/src/web-ui/src/infrastructure/markdown/SessionMarkdownImage.tsx @@ -6,10 +6,11 @@ import { getActiveSurfaceScope, onSurfaceActivated } from '@/infrastructure/peer export type SessionImageReader = (path: string, refresh?: boolean) => Promise; /** Session providers own these bytes; neither loading nor fallback may read local files. */ -export function SessionMarkdownImage({ path, alt, title, read, download, onPreview }: { +export function SessionMarkdownImage({ path, alt, title, read, download, onPreview, thumbnail }: { path: string; alt?: string; title?: string; read: SessionImageReader; download?: (path: string) => Promise; onPreview?: (source: string, alt?: string) => void; + thumbnail?: boolean; }) { const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); const [attempt, setAttempt] = useState(0); @@ -34,6 +35,13 @@ export function SessionMarkdownImage({ path, alt, title, read, download, onPrevi const source = current?.src; if (source) return {alt { + if (!onPreview || event.currentTarget.closest('a, button') || !['Enter', ' '].includes(event.key)) return; + event.preventDefault(); + event.stopPropagation(); + onPreview(source, alt || undefined); + }} onClick={(event) => { // An image owned by a link or a file link keeps that owner's behavior. if (!onPreview || event.currentTarget.closest('a, button')) return; diff --git a/src/web-ui/src/infrastructure/markdown/rehypeImageThumbnails.ts b/src/web-ui/src/infrastructure/markdown/rehypeImageThumbnails.ts new file mode 100644 index 0000000000..5217074be6 --- /dev/null +++ b/src/web-ui/src/infrastructure/markdown/rehypeImageThumbnails.ts @@ -0,0 +1,65 @@ +type HtmlNode = { + type: string; + tagName?: string; + value?: string; + properties?: Record; + children?: HtmlNode[]; +}; + +/** Group adjacent image-only blocks in chat, leaving prose and linked badges intact. */ +export function rehypeImageThumbnails({ enabled = false }: { enabled?: boolean } = {}) { + return (tree: HtmlNode) => { + if (!enabled) return; + + const standaloneImages = (node: HtmlNode): HtmlNode[] | undefined => { + if (node.type !== 'element') return; + if (node.tagName === 'img') return [node]; + if (node.tagName !== 'p') return; + const images: HtmlNode[] = []; + for (const child of node.children ?? []) { + if (child.type === 'text' && !child.value?.trim()) continue; + if (child.type === 'element' && child.tagName === 'br') continue; + if (child.type !== 'element' || child.tagName !== 'img') return; + images.push(child); + } + return images.length ? images : undefined; + }; + + const group = (parent: HtmlNode) => { + if (!parent.children || (parent.type === 'element' && !['div', 'blockquote', 'li', 'td', 'th', 'section', 'article', 'details', 'figure'].includes(parent.tagName ?? ''))) return; + const children: HtmlNode[] = []; + let images: HtmlNode[] = []; + const flush = () => { + if (!images.length) return; + for (const image of images) { + image.properties ??= {}; + const classes = image.properties.className; + image.properties.className = [...(Array.isArray(classes) ? classes : classes ? [String(classes)] : []), 'markdown-image-thumbnail']; + } + children.push({ + type: 'element', tagName: 'div', + properties: { className: [ + 'markdown-image-gallery', + ...(images.length === 1 ? ['markdown-image-gallery--single'] : []), + ...([2, 4].includes(images.length) ? ['markdown-image-gallery--paired'] : []), + ] }, + children: images.map(image => ({ + type: 'element', tagName: 'span', properties: { className: ['markdown-image-tile'] }, children: [image], + })), + }); + images = []; + }; + for (const node of parent.children) { + const found = standaloneImages(node); + if (found) { images.push(...found); continue; } + if (images.length && node.type === 'text' && !node.value?.trim()) continue; + flush(); + if (node.type === 'element') group(node); + children.push(node); + } + flush(); + parent.children = children; + }; + group(tree); + }; +} diff --git a/src/web-ui/src/locales/en-US/scenes/capabilities.json b/src/web-ui/src/locales/en-US/scenes/capabilities.json index 132bd468b1..934df3fdbb 100644 --- a/src/web-ui/src/locales/en-US/scenes/capabilities.json +++ b/src/web-ui/src/locales/en-US/scenes/capabilities.json @@ -42,6 +42,7 @@ "session-lifecycle": "Rename, archive, restore, delete, copy, fork, and export sessions", "model-mode-permissions": "Choose the model, work mode, memory mode, and permission mode per session", "creative-frontend-authoring": "Extend client UI and runtime capabilities in Creative mode", + "image-generation": "Generate images with the configured default image model", "turn-control": "Start, steer, interrupt, cancel, or recover an agent turn", "permission-mailbox": "Review and answer pending permission requests individually or in a batch", "history-search": "Page through, search, and backfill historical turns and session content", @@ -467,7 +468,7 @@ "available-models": "Let an agent inspect models available under the current configuration", "model-catalog": "Inspect models.dev catalog status, refresh it manually, and reveal its local cache", "advanced-request": "Add custom headers, request-body fields, and SSL behavior per provider", - "connection-test": "Test provider or model connectivity before saving and refresh the runtime client", + "connection-test": "Test model connections or image generation and refresh the runtime client", "model-pools": "Manage model pools, routing strategies, and session model selection" } }, diff --git a/src/web-ui/src/locales/zh-CN/scenes/capabilities.json b/src/web-ui/src/locales/zh-CN/scenes/capabilities.json index 1cec6eca7d..7ed31aa627 100644 --- a/src/web-ui/src/locales/zh-CN/scenes/capabilities.json +++ b/src/web-ui/src/locales/zh-CN/scenes/capabilities.json @@ -42,6 +42,7 @@ "session-lifecycle": "重命名、归档、恢复、删除、复制、分叉与导出会话", "model-mode-permissions": "为会话选择模型、工作模式、记忆模式与权限模式", "creative-frontend-authoring": "让创造模式扩展客户端界面与运行时能力", + "image-generation": "通过已配置的默认图像模型生成图片", "turn-control": "开始、引导、打断、取消或恢复一次智能体运行", "permission-mailbox": "查看并逐个或批量回答等待中的权限请求", "history-search": "分页加载、搜索和回填历史消息与会话内容", @@ -467,7 +468,7 @@ "available-models": "让智能体查看当前配置下可用的模型", "model-catalog": "查看 models.dev 目录状态、手动刷新并定位本地缓存", "advanced-request": "为供应商添加自定义 Header、请求体字段与 SSL 行为", - "connection-test": "保存前测试供应商或模型连接并刷新运行时客户端", + "connection-test": "测试模型连接或图像生成，并刷新运行时客户端", "model-pools": "管理模型池、路由策略与会话模型选择" } }, diff --git a/src/web-ui/src/locales/zh-TW/scenes/capabilities.json b/src/web-ui/src/locales/zh-TW/scenes/capabilities.json index a928151654..af36fa3185 100644 --- a/src/web-ui/src/locales/zh-TW/scenes/capabilities.json +++ b/src/web-ui/src/locales/zh-TW/scenes/capabilities.json @@ -42,6 +42,7 @@ "session-lifecycle": "重命名、歸檔、恢復、刪除、複製、分叉與導出工作階段", "model-mode-permissions": "為工作階段選擇模型、工作模式、記憶模式與權限模式", "creative-frontend-authoring": "讓創造模式擴展客戶端界面與運行時能力", + "image-generation": "透過已設定的預設圖像模型生成圖片", "turn-control": "開始、引導、打斷、取消或恢復一次智能體運行", "permission-mailbox": "查看並逐個或批量回答等待中的權限請求", "history-search": "分頁加載、搜索和回填歷史消息與工作階段內容", @@ -467,7 +468,7 @@ "available-models": "讓智能體查看當前配置下可用的模型", "model-catalog": "查看 models.dev 目錄狀態、手動重新整理並定位本地緩存", "advanced-request": "為供應商添加自定義 Header、請求體字段與 SSL 行為", - "connection-test": "保存前測試供應商或模型連接並重新整理運行時客戶端", + "connection-test": "測試模型連線或圖像生成，並重新整理執行階段用戶端", "model-pools": "管理模型池、路由策略與工作階段模型選擇" } },