From fda7a7b1553b0904710455b5bc66a0395e8cc77a Mon Sep 17 00:00:00 2001 From: wgqqqqq Date: Mon, 26 Oct 2026 10:51:25 +0800 Subject: [PATCH 1/9] feat(miniapps): add lifecycle-safe MiniAppBrowser control --- docs/interactive-capabilities/README.md | 16 +- .../capabilities.json | 80 +- .../capabilities/feature.miniapps.md | 2 + .../technical/product-control-open-audit.json | 2 +- .../technical/tauri-command-map.json | 38 +- src/apps/desktop/src/lib.rs | 4 + .../desktop/src/miniapp_desktop/AGENTS.md | 49 + src/apps/desktop/src/miniapp_desktop/mod.rs | 711 +++++++- src/apps/desktop/src/miniapp_web_host.rs | 741 ++++++++ src/apps/miniapp-shell/AGENTS.md | 33 + src/apps/miniapp-shell/src/shell.html | 151 +- src/apps/miniapp-shell/src/shell.rs | 7 +- src/crates/adapters/miniapp-ipc/src/lib.rs | 6 + .../miniapp-dev/api-reference.md | 50 +- .../agents/definitions/modes/creative.rs | 9 + .../assembly/core/src/agentic/agents/mod.rs | 5 + .../tools/implementations/control_hub_tool.rs | 11 +- .../miniapp_browser/actions.rs | 1506 +++++++++++++++++ .../implementations/miniapp_browser/mod.rs | 342 ++++ .../implementations/miniapp_browser/result.rs | 236 +++ .../src/agentic/tools/implementations/mod.rs | 4 + .../core/src/agentic/tools/miniapp_surface.rs | 758 +++++++++ .../assembly/core/src/agentic/tools/mod.rs | 2 + .../tools/product_runtime/materialization.rs | 3 + .../core/src/agentic/tools/registry.rs | 2 + .../src/agentic/tools/tool_context_runtime.rs | 19 + .../generated/product-control-catalog.json | 80 +- .../generated/remote-surface-registry.json | 26 +- .../src/miniapp/bridge_builder.rs | 299 ++++ .../product-domains/src/miniapp/mod.rs | 1 + .../src/miniapp/web_surface.rs | 1193 +++++++++++++ .../src/remote_surface/table.rs | 2 + .../tests/miniapp_contracts.rs | 46 + .../execution/tool-contracts/src/framework.rs | 5 + .../execution/tool-provider-groups/src/lib.rs | 6 +- .../interactive-capabilities/catalog.json | 28 + .../generated/interactive-capabilities.json | 80 +- .../miniapps/agent/MiniAppAgentSurface.tsx | 10 + .../miniapps/agent/miniAppAgentBridge.test.ts | 212 +++ .../miniapps/agent/miniAppAgentBridge.ts | 263 +++ .../agent/miniAppAgentCapture.test.ts | 127 ++ .../miniapps/agent/miniAppAgentCapture.ts | 195 +++ .../agent/miniAppAgentDomRuntime.test.ts | 227 +++ .../miniapps/agent/miniAppAgentDomRuntime.ts | 744 ++++++++ .../miniapps/agent/miniAppAgentProtocol.ts | 169 ++ .../miniapps/agent/miniAppAgentRouter.test.ts | 298 ++++ .../miniapps/agent/miniAppAgentRouter.ts | 282 +++ .../agent/miniAppAgentSurfaceRegistry.ts | 96 ++ .../agent/useMiniAppAgentSurface.test.tsx | 109 ++ .../miniapps/agent/useMiniAppAgentSurface.ts | 119 ++ .../components/MiniAppRunner.test.tsx | 39 + .../miniapps/components/MiniAppRunner.tsx | 58 +- .../miniapps/components/MiniAppShell.test.ts | 107 ++ .../src/app/startup/MainApplicationRoot.tsx | 2 + .../api/generated/productControl.ts | 2 +- .../api/generated/remoteSurface.ts | 4 +- .../MiniAppAgentSurfaceAPI.test.ts | 25 + .../api/service-api/MiniAppAgentSurfaceAPI.ts | 53 + src/web-ui/src/locales/en-US/flow-chat.json | 1 + src/web-ui/src/locales/zh-CN/flow-chat.json | 1 + src/web-ui/src/locales/zh-TW/flow-chat.json | 1 + 61 files changed, 9615 insertions(+), 82 deletions(-) create mode 100644 src/apps/desktop/src/miniapp_web_host.rs create mode 100644 src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/actions.rs create mode 100644 src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/mod.rs create mode 100644 src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/result.rs create mode 100644 src/crates/assembly/core/src/agentic/tools/miniapp_surface.rs create mode 100644 src/crates/contracts/product-domains/src/miniapp/web_surface.rs create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/MiniAppAgentSurface.tsx create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.test.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.test.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.test.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentProtocol.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.test.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentSurfaceRegistry.ts create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.test.tsx create mode 100644 src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.ts create mode 100644 src/web-ui/src/infrastructure/api/service-api/MiniAppAgentSurfaceAPI.test.ts create mode 100644 src/web-ui/src/infrastructure/api/service-api/MiniAppAgentSurfaceAPI.ts diff --git a/docs/interactive-capabilities/README.md b/docs/interactive-capabilities/README.md index 057976a136..b569b2ebd8 100644 --- a/docs/interactive-capabilities/README.md +++ b/docs/interactive-capabilities/README.md @@ -1,9 +1,9 @@ # OpenBitFun 功能与设置目录 / OpenBitFun Features & Settings -OpenBitFun Playbook 当前包含 **23 个功能**和 **21 个设置页**，共 **44 个**用户可理解的条目、**333 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 +OpenBitFun Playbook 当前包含 **23 个功能**和 **21 个设置页**，共 **44 个**用户可理解的条目、**334 项**有源码证据的子能力。每个条目有独立 Markdown，并直接服务于说明书网站、OpenBitFun 全局搜索和 `OpenBitFunControl` 智能体工具。 -OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, and **333** source-backed sub-capabilities across **44** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. +OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, and **334** source-backed sub-capabilities across **44** user-facing entries. Every entry has its own Markdown page and directly powers the website, in-app global search, and the `OpenBitFunControl` agent tool. ## 唯一事实源 / Single source of truth @@ -27,20 +27,20 @@ OpenBitFun Playbook currently contains **23 features**, **21 settings pages**, a - Generated per-item interaction audit: `docs/interactive-capabilities/technical/product-control-open-audit.json` - Generated low-level audit map: `docs/interactive-capabilities/technical/tauri-command-map.json` -说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **701** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 +说明书、网站、搜索和智能体只看“功能 + 设置 + 子能力”。每项子能力都必须引用已注册 Tauri Command 或可解析的源码标记；这些证据不会进入公开目录。当前 **703** 个 Tauri 命令只用于实现覆盖审计。产品 UI 交互源码会在生成和检查时扫描并校验，但不会保存成随普通 UI 改动频繁变化的版本化快照。 -Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **701** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. +Docs, website, search, and agents see only features, settings, and documented sub-capabilities. Every sub-capability must reference a registered Tauri command or a resolvable source marker; evidence is stripped from public projections. The **703** Tauri commands remain implementation-audit evidence only. Product UI interaction sources are scanned and validated during generation and checks, but are not stored as a versioned snapshot that churns with ordinary UI changes. ## 控制边界 / Control boundary -- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **52**、委托 **65**、需交互 **216**、不支持 **0**。 +- 每个子能力都明确标记为直接控制、委托给专用智能体工具、需交互打开或不支持；“打开页面”不会再被统计成“智能体已控制”。当前覆盖：直接 **52**、委托 **66**、需交互 **216**、不支持 **0**。 - 稳定行为声明为带 JSON 输入契约的 `operations` 或 `options`，并绑定原生产品控制 Provider；智能体不接触原始 Tauri Command。 -- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 333 项子能力都不会写入 system prompt。 +- `OpenBitFunControl list` 和 `search` 都返回带 `nextCursor` 的精简分页结果；目录可持续增长，不靠固定总量上限。完整目录和 334 项子能力都不会写入 system prompt。 - 目录发现与契约读取不依赖 React 或可见窗口。普通配置型 option 统一由 Product Assembly 的共享 ConfigService 执行器读、写并回读，因此 Desktop、CLI 与 Headless 表面走同一份实现；只有宿主原生 operation/provider option 和界面导航按表面注册适配器，缺失时必须明确返回不可用，禁止静默回退本机。只读智能体只能发现和读取目录。 -- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **52 direct**, **65 delegated**, **216 interactive**, and **0 unsupported**. +- Every documented item is classified as direct control, delegated Agent control, interactive opening, or unsupported; opening a page is never counted as direct control. Current coverage is **52 direct**, **66 delegated**, **216 interactive**, and **0 unsupported**. - Stable behavior becomes a typed `operation` or `option` with a JSON input contract and a native product-control provider. Agents never receive raw Tauri commands. -- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 333 documented items enters the system prompt. +- `OpenBitFunControl list` and `search` return compact pages with a `nextCursor`; the catalog can grow without a fixed total-size ceiling. Neither the full catalog nor its 334 documented items enters the system prompt. - Discovery and contract lookup do not depend on React or a visible window. Ordinary config-backed options are read, written, and read back by one Product Assembly ConfigService executor shared by Desktop, CLI, and headless surfaces. Only host-native operations/provider options and presentation routes install surface adapters; missing adapters return explicit unavailability without local fallback. Read-only agents may only discover and inspect entries. ## 防腐化门禁 / Anti-drift gates diff --git a/docs/interactive-capabilities/capabilities.json b/docs/interactive-capabilities/capabilities.json index 5fde780727..9a76c0e6ad 100644 --- a/docs/interactive-capabilities/capabilities.json +++ b/docs/interactive-capabilities/capabilities.json @@ -4,7 +4,7 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "de9d03ab0cbb5268869e61d1a3e363c5210d93b9dccb2c8caccc99006074732a", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ { @@ -147,10 +147,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -5765,6 +5765,7 @@ "versions", "agent-authoring", "agent-market-submit", + "agent-operate-app", "publishing" ], "kind": "query", @@ -6711,6 +6712,51 @@ "actionId": "surface.miniapps.open" } }, + { + "id": "feature.miniapps:delegate:agent-operate-app", + "capabilityId": "feature.miniapps", + "itemIds": [ + "agent-operate-app" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "MiniAppBrowser" + ], + "presentationTarget": { + "kind": "action", + "actionId": "surface.miniapps.open" + } + }, { "id": "feature.miniapps:open:publishing", "capabilityId": "feature.miniapps", @@ -22950,6 +22996,25 @@ ] } }, + { + "id": "agent-operate-app", + "titleZh": "由智能体直接操作 Mini App 界面", + "titleEn": "Let an agent operate a Mini App interface directly", + "control": { + "kind": "delegate", + "tools": [ + "MiniAppBrowser" + ], + "workflowZh": [ + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态" + ], + "workflowEn": [ + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" + ] + } + }, { "id": "publishing", "titleZh": "撤回和审查市场发布，并可渲染幻灯片页面", @@ -23295,6 +23360,8 @@ "Let an agent initialize, author, and validate a Mini App", "在用户明确要求时由智能体提交已完成的 Mini App 到市场审核", "Let an agent submit a completed Mini App for market review when explicitly requested", + "由智能体直接操作 Mini App 界面", + "Let an agent operate a Mini App interface directly", "撤回和审查市场发布，并可渲染幻灯片页面", "Withdraw and review market releases, and render slide pages", "打开 Mini Apps", @@ -23320,7 +23387,12 @@ "仅在用户明确要求对外提交时调用 PublishMiniApp，并提供已完成应用、许可与发布说明", "登录、校验或上传返回可恢复状态时，向用户展示具体下一步，不把待授权或待审核状态报告成发布成功", "Call PublishMiniApp only after an explicit request to submit externally, with the completed app, license, and release notes", - "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published" + "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published", + "MiniAppBrowser", + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态", + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" ], "docsUrl": "https://playbook.openbitfun.com/capabilities/feature.miniapps/" }, diff --git a/docs/interactive-capabilities/capabilities/feature.miniapps.md b/docs/interactive-capabilities/capabilities/feature.miniapps.md index 68121e2cd6..9ae124de47 100644 --- a/docs/interactive-capabilities/capabilities/feature.miniapps.md +++ b/docs/interactive-capabilities/capabilities/feature.miniapps.md @@ -39,6 +39,8 @@ Install, create, and run lightweight apps inside OpenBitFun, combining UI, data, - Let an agent initialize, author, and validate a Mini App - **由专用智能体工具控制 / Delegated Agent tool** · `PublishMiniApp` · 在用户明确要求时由智能体提交已完成的 Mini App 到市场审核 - Let an agent submit a completed Mini App for market review when explicitly requested +- **由专用智能体工具控制 / Delegated Agent tool** · `MiniAppBrowser` · 由智能体直接操作 Mini App 界面 + - Let an agent operate a Mini App interface directly - **智能体可定位入口，需交互完成 / Agent opens; interaction required** · 撤回和审查市场发布，并可渲染幻灯片页面 - Withdraw and review market releases, and render slide pages diff --git a/docs/interactive-capabilities/technical/product-control-open-audit.json b/docs/interactive-capabilities/technical/product-control-open-audit.json index 602eccb317..3393f377e0 100644 --- a/docs/interactive-capabilities/technical/product-control-open-audit.json +++ b/docs/interactive-capabilities/technical/product-control-open-audit.json @@ -1,7 +1,7 @@ { "schemaVersion": 1, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "catalogDigest": "de9d03ab0cbb5268869e61d1a3e363c5210d93b9dccb2c8caccc99006074732a", "count": 216, "reasonCounts": { "externalAuth": 4, diff --git a/docs/interactive-capabilities/technical/tauri-command-map.json b/docs/interactive-capabilities/technical/tauri-command-map.json index 8e7c9ceeea..25e305e4e2 100644 --- a/docs/interactive-capabilities/technical/tauri-command-map.json +++ b/docs/interactive-capabilities/technical/tauri-command-map.json @@ -1,13 +1,13 @@ { "schemaVersion": 2, "generatedFrom": "src/shared/interactive-capabilities/catalog.json", - "catalogDigest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", - "commandCount": 701, + "catalogDigest": "de9d03ab0cbb5268869e61d1a3e363c5210d93b9dccb2c8caccc99006074732a", + "commandCount": 703, "coverage": { - "commandCount": 701, + "commandCount": 703, "documentedCommandCount": 644, - "implementationCommandCount": 57, - "implementationDigest": "503b48eebc37750e50e0427d861e884dfa353a81aae32236e4bc2d7e0f5310e2" + "implementationCommandCount": 59, + "implementationDigest": "0bce9fbda3e8b517e409aec124ca0f2c3b36a7954d996400d9ccf405194dc768" }, "commands": [ { @@ -5877,6 +5877,20 @@ "signature": "fn manage_dialog_queue( runtime: State<'_, DesktopRuntimeContext>, request: openbitfun_runtime_ports::DialogQueueRequest, ) -> Result", "remoteWorkspacePolicy": "RemoteRouted" }, + { + "id": "mark_miniapp_web_surface_ready", + "moduleId": "miniapp_web_host", + "capabilityId": "feature.miniapps", + "capabilityIds": [ + "feature.miniapps" + ], + "documentedItemIds": [], + "visibility": "implementation", + "rustPath": "miniapp_web_host::mark_miniapp_web_surface_ready", + "sourceFile": "src/apps/desktop/src/miniapp_web_host.rs", + "signature": "fn mark_miniapp_web_surface_ready(request: Option)", + "remoteWorkspacePolicy": "WorkspaceAgnostic" + }, { "id": "mark_openbitfun_control_surface_ready", "moduleId": "openbitfun_control_host", @@ -8006,6 +8020,20 @@ "signature": "fn report_ide_control_result(request: IdeControlResultRequest) -> Result<(), String>", "remoteWorkspacePolicy": "LegacyUnaudited" }, + { + "id": "report_miniapp_web_result", + "moduleId": "miniapp_web_host", + "capabilityId": "feature.miniapps", + "capabilityIds": [ + "feature.miniapps" + ], + "documentedItemIds": [], + "visibility": "implementation", + "rustPath": "miniapp_web_host::report_miniapp_web_result", + "sourceFile": "src/apps/desktop/src/miniapp_web_host.rs", + "signature": "fn report_miniapp_web_result( request: ReportMiniAppWebResultRequest, ) -> Result<(), String>", + "remoteWorkspacePolicy": "WorkspaceAgnostic" + }, { "id": "report_openbitfun_control_result", "moduleId": "openbitfun_control_host", diff --git a/src/apps/desktop/src/lib.rs b/src/apps/desktop/src/lib.rs index 265bcba307..1b4f79ffb2 100644 --- a/src/apps/desktop/src/lib.rs +++ b/src/apps/desktop/src/lib.rs @@ -35,6 +35,7 @@ mod macos_window_chrome; #[cfg(target_os = "macos")] mod macos_window_close; mod miniapp_desktop; +mod miniapp_web_host; // Keep one macro expansion: on macOS it emits a process-wide Info.plist symbol. fn desktop_context() -> tauri::Context { @@ -1136,6 +1137,7 @@ pub async fn run() { openbitfun_webdriver::maybe_start(app_handle.clone()); builtin_browser_host::install(app_handle.clone()); openbitfun_control_host::install(app_handle.clone()); + miniapp_web_host::install(app_handle.clone()); if !miniapp_background { if let Err(error) = desktop_cards::install(&app_handle) { log::warn!("Desktop cards unavailable: {}", error); @@ -1492,6 +1494,8 @@ pub async fn run() { openbitfun_control_host::mark_openbitfun_control_surface_unready, openbitfun_control_host::product_control_invoke, openbitfun_control_host::report_openbitfun_control_result, + miniapp_web_host::mark_miniapp_web_surface_ready, + miniapp_web_host::report_miniapp_web_result, get_health_status, get_statistics, test_ai_connection, diff --git a/src/apps/desktop/src/miniapp_desktop/AGENTS.md b/src/apps/desktop/src/miniapp_desktop/AGENTS.md index 434c889de2..7d578f0fc9 100644 --- a/src/apps/desktop/src/miniapp_desktop/AGENTS.md +++ b/src/apps/desktop/src/miniapp_desktop/AGENTS.md @@ -51,6 +51,43 @@ Content action contracts are reserved and their advertised capability list is empty. Remote workspace, Peer, mobile, and detached hosts do not acquire local presentation as a silent fallback. +The agent bridge is relayed, never reimplemented. A Shell that advertises +`miniapp_agent_bridge_v1` lets the host push `agentRequest` events into the +window; the Shell forwards them to its content frame over the same postMessage +protocol a tab uses and answers with `agentResponse`, so one MiniApp answers +identically wherever it is presented. `agent_window` routes a call only to a +ready Window whose Shell advertised the capability, and only when the command is +bound to that instance; every other presentation stays with the Web UI tab +runtime. DOM, pixel, and synthetic-input actions are refused on a window +presentation instead of being faked. Polls are bounded long polls woken by the +lease's `notify`, so a relayed request costs one IPC round trip rather than a +heartbeat; keep that bound and keep `poll` answering `released` when the lease is +gone. + +Agent requests require an authenticated `agentClaim` before the Shell forwards +input or bridge calls. The host binds each waiter to its view, monotonic deadline +and single-use claim; completed, timed-out, canceled and replayed requests cannot +claim execution. The Shell checks the delivery expiry and document generation +after claiming, and rejects pending requests when replacing content. Keep the +poll cursor across content reloads of the same host/view; consume a reload event +before restarting. Host and Shell are upgraded together for this protocol. + +A window answers for two different ids, and they are not interchangeable: the +`view_id` identifies the view lease, the `instance_id` identifies the running +app and is what `open` reports and what the agent session must bind to. Keep them +in the named `AgentWindow` struct rather than a positional tuple; swapping them +made every bridge call miss its window and silently fall through to the tab +runtime. + +Choosing a presentation is Desktop policy and lives in the Desktop Web surface +adapter, never in the portable contract: a live window of the app first, then a +tab already showing it, then the app the user published to the desktop — which +is launched as a normal app and waited for, bounded by `WINDOW_BOOT_TIMEOUT` — +and only then a new tab. Launch or registration failure is a logged degradation +to the tab, not an error, because the in-product surface still works. Never +launch a published app when the app is already on screen; that would hand the +user a second copy of the state they are looking at. + Focused checks (no browser/computer-use validation): For custom chrome, also run `pnpm --dir src/web-ui run test:run src/app/scenes/miniapps/MiniAppScene.chrome.test.tsx src/app/scenes/miniapps/components/MiniAppTitlebar.test.tsx src/app/scenes/miniapps/components/MiniAppShell.test.ts` @@ -59,6 +96,7 @@ and `cargo test --locked -p openbitfun-desktop --lib openbitfun_control_host::mi ```bash cargo test --locked -p openbitfun-desktop --lib miniapp_desktop +cargo test --locked -p openbitfun-desktop --lib miniapp_web_host cargo test --locked -p openbitfun-desktop --lib remote_workspace_policy cargo build -p openbitfun-desktop pnpm --dir src/web-ui run test:run src/app/scenes/miniapps/miniAppPresentationStore.test.ts src/app/scenes/miniapps/miniAppLifecycle.test.ts src/app/scenes/miniapps/components/MiniAppRunner.test.tsx src/app/scenes/miniapps/hooks/useMiniAppBridge.test.tsx @@ -68,6 +106,17 @@ pnpm --dir src/web-ui run test:run src/app/scenes/miniapps/components/MiniAppSys pnpm --dir src/web-ui run test:run src/app/scenes/miniapps/MiniAppPresence.test.tsx ``` +For Agent control, document-generation handles, strict-frame readiness, bridge +payload bounds and Peer activation fences, run: + +```bash +pnpm --dir src/web-ui run test:run src/app/scenes/miniapps/agent src/app/scenes/miniapps/components/MiniAppRunner.test.tsx src/infrastructure/api/service-api/MiniAppAgentSurfaceAPI.test.ts +cargo test --locked -p openbitfun-core --no-default-features --features agent-runtime,git,tools-miniapp --lib agentic::tools::miniapp_surface::tests +``` + +These fixtures cover local DOM, transport and activation state; they do not +establish native WebView rendering or live SSH, relay, Peer or dispatch behavior. + Native bundle fixture tests must write only to their temporary directory and must not launch UI. macOS fixture evidence does not prove Windows taskbar or Linux compositor behavior; report which platform was actually exercised. diff --git a/src/apps/desktop/src/miniapp_desktop/mod.rs b/src/apps/desktop/src/miniapp_desktop/mod.rs index 8e6c138890..ea8d1a4b8e 100644 --- a/src/apps/desktop/src/miniapp_desktop/mod.rs +++ b/src/apps/desktop/src/miniapp_desktop/mod.rs @@ -128,6 +128,9 @@ pub(crate) struct DesktopMiniApps { endpoint: Mutex>, socket_dir: Mutex>, worker_listeners: Mutex>, + /// Agent bridge calls waiting on a Shell relay. Bounded, and every entry is + /// removed by its waiter, so a Shell that never answers leaks nothing. + agents: Mutex>, } impl DesktopMiniApps { @@ -146,6 +149,264 @@ impl DesktopMiniApps { .or_default() .clone() } + + /// The live window presentation of `app_id`, when that window can answer + /// agent-bridge requests. Windows the Shell never advertised a bridge for are + /// skipped on purpose: routing to them would only produce a timeout. + pub(crate) fn agent_window(&self, app_id: &str) -> Option { + let snapshot = self.instances.snapshot(); + let shells = self.shells.lock().unwrap_or_else(|p| p.into_inner()); + snapshot.instances.iter().find_map(|instance| { + if instance.app_id != app_id { + return None; + } + let view = instance.primary_view.as_ref()?; + if view.kind != MiniAppViewKind::Window { + return None; + } + let lease = shells.get(&view.view_id)?; + match lease.agent_bridge { + true => Some(AgentWindow { + view_id: view.view_id.clone(), + instance_id: instance.instance_id.clone(), + }), + false => None, + } + }) + } + + /// `true` when this app is already on screen in the product window, so an + /// Agent `open` should keep that presentation instead of launching another. + pub(crate) fn tab_presentation(&self, app_id: &str) -> Option { + self.instances + .snapshot() + .instances + .iter() + .find_map(|instance| { + if instance.app_id != app_id { + return None; + } + let view = instance.primary_view.as_ref()?; + (view.kind == MiniAppViewKind::Tab).then(|| instance.instance_id.clone()) + }) + } + + /// Relay one agent-bridge request into a window and wait for its answer. + /// + /// The window owns the frame, so the request travels as an event the Shell + /// polls for and the answer comes back over the same authenticated IPC call + /// that every other Shell operation uses. Nothing here evaluates JavaScript. + pub(crate) async fn call_agent_bridge( + &self, + app_id: &str, + method: &str, + action: Option, + timeout: std::time::Duration, + ) -> Result { + let Some(window) = self.agent_window(app_id) else { + return Err(format!( + "SURFACE_UNAVAILABLE: MiniApp {app_id} has no windowable agent bridge" + )); + }; + let expires_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|error| format!("HOST_ERROR: {error}"))? + .as_millis() + .saturating_add(timeout.as_millis()); + let request_id = format!( + "agent-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or(0) + ); + // Shape the request exactly as the injected frame runtime reads it, so the + // relay stays a relay: only the transport differs from a tab. + let mut relayed = json!({ + "requestId": request_id, + "method": method, + "expiresAt": expires_at as u64, + "schemaVersion": openbitfun_product_domains::miniapp::web_surface::MINIAPP_WEB_SCHEMA_VERSION, + }); + if let Some(action) = action { + relayed["action"] = action; + } + if relayed.to_string().len() > ipc::MINIAPP_AGENT_MESSAGE_MAX_BYTES { + return Err("INVALID_PARAMS: the agent request payload is too large".into()); + } + let (sender, receiver) = tokio::sync::oneshot::channel(); + { + let mut agents = self.agents.lock().unwrap_or_else(|p| p.into_inner()); + // Prune canceled callers without dropping other live requests. + agents.retain(|_, pending| { + !pending.sender.is_closed() && std::time::Instant::now() < pending.deadline + }); + agents.insert( + request_id.clone(), + PendingAgentRequest { + sender, + view_id: window.view_id.clone(), + deadline: std::time::Instant::now() + timeout, + claimed: false, + }, + ); + } + let pushed = { + let mut shells = self.shells.lock().unwrap_or_else(|p| p.into_inner()); + match shells.get_mut(&window.view_id) { + Some(lease) => { + lease.push("agentRequest", relayed); + true + } + None => false, + } + }; + if !pushed { + self.agents + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&request_id); + return Err("SURFACE_UNAVAILABLE: the MiniApp window is no longer open".into()); + } + match tokio::time::timeout(timeout, receiver).await { + Ok(Ok(answer)) => answer, + Ok(Err(_)) => Err("BRIDGE_TIMEOUT: the MiniApp window dropped the request".into()), + Err(_) => { + self.agents + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&request_id); + Err("BRIDGE_TIMEOUT: the MiniApp window did not answer in time".into()) + } + } + } + + /// Claim a live request once, using the authenticated Shell view identity. + fn claim_agent_request(&self, view_id: &str, request_id: &str) -> bool { + self.agents + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_mut(request_id) + .is_some_and(|pending| pending.claim(view_id)) + } + + /// Only the view that claimed the request may settle its result. + fn settle_agent_bridge(&self, view_id: &str, answer: Value) -> Result<(), String> { + let request_id = answer + .get("requestId") + .and_then(Value::as_str) + .ok_or("The agent answer is missing its requestId")?; + let pending = { + let mut agents = self.agents.lock().unwrap_or_else(|p| p.into_inner()); + let pending = agents + .get(request_id) + .ok_or("The agent request is no longer pending")?; + if pending.view_id != view_id { + return Err("The agent answer belongs to another view".into()); + } + if !pending.claimed { + return Err("The agent request has not been claimed".into()); + } + agents.remove(request_id).expect("checked pending request") + }; + let payload = if answer.to_string().len() > ipc::MINIAPP_AGENT_MESSAGE_MAX_BYTES { + Err("BRIDGE_REJECTED: the MiniApp bridge response exceeds the 32 KiB limit".to_string()) + } else { + match answer.get("ok").and_then(Value::as_bool) { + Some(true) => Ok(BridgeAnswer { + value: answer.get("value").cloned().unwrap_or(Value::Null), + capabilities: declared_names(&answer, "capabilities", 8), + actions: declared_names(&answer, "actions", 32), + }), + // A bare message comes from the MiniApp itself, so it is a refusal. The + // Shell names its own transport failure in `code`, which is why the + // failure kind is data here instead of something parsed out of prose. + _ => Err({ + use openbitfun_product_domains::miniapp::web_surface::MiniAppWebErrorCode; + let message = answer + .get("error") + .and_then(Value::as_str) + .map(str::trim) + .filter(|message| !message.is_empty()) + .unwrap_or("the MiniApp refused the agent request"); + let code = match answer.get("code").and_then(Value::as_str) { + Some("BRIDGE_TIMEOUT") => MiniAppWebErrorCode::BridgeTimeout, + Some("SURFACE_UNAVAILABLE") => MiniAppWebErrorCode::SurfaceUnavailable, + Some("INVALID_PARAMS") => MiniAppWebErrorCode::InvalidParams, + _ => MiniAppWebErrorCode::BridgeRejected, + }; + format!("{}: {message}", code.as_str()) + }), + } + }; + pending + .sender + .send(payload) + .map_err(|_| "The agent request receiver is gone".to_string()) + } +} + +/// Test scaffolding for the transport-routing rules that live in the Desktop +/// adapters: they need a live window without launching one. +#[cfg(test)] +impl DesktopMiniApps { + pub(crate) fn test_window(&mut self, app_id: &str) -> AgentWindow { + let (instance, view) = self + .instances + .prepare(app_id.to_string(), None, MiniAppViewKind::Window) + .expect("window view"); + self.shells + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert( + view.view_id.clone(), + ShellLease { + instance_id: instance.instance_id.clone(), + nonce: "nonce".to_string(), + streams: HashSet::new(), + events: VecDeque::new(), + sequence: 0, + event_bytes: 0, + agent_bridge: true, + notify: Arc::new(tokio::sync::Notify::new()), + }, + ); + self.instances + .ready(&instance.instance_id, &view.view_id) + .expect("view becomes primary"); + AgentWindow { + view_id: view.view_id, + instance_id: instance.instance_id, + } + } +} + +/// A live window presentation of one app. +/// +/// The two ids answer different questions — a view is where the request is +/// delivered, an instance is what a session addresses — and they used to travel +/// as a positional pair. A caller that swapped them looked correct and silently +/// routed every bridge request to another surface, so they are named here. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct AgentWindow { + pub(crate) view_id: String, + pub(crate) instance_id: String, +} + +/// One relayed bridge answer plus what the document declared about itself. +/// +/// The declaration travels with every answer because the MiniApp can re-declare +/// at any time (`expose()` replaces the handlers, a restart re-anchors), so a +/// cached copy anywhere above this line would go stale silently. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct BridgeAnswer { + pub(crate) value: Value, + /// Capability names the document last announced: `auto-observe`, `observe`, `act`. + pub(crate) capabilities: Vec, + /// Action types the document listed for `act`. Empty means it listed none, + /// which is unknown rather than "accepts nothing". + pub(crate) actions: Vec, } struct ShellLease { @@ -155,6 +416,72 @@ struct ShellLease { events: VecDeque, sequence: u64, event_bytes: usize, + /// The Shell advertised the agent bridge capability, so the host may relay + /// observe/act into this window. An older Shell leaves this false. + agent_bridge: bool, + /// Wakes a waiting `poll` the moment an event is pushed. Polls are otherwise + /// long-polled with a bounded wait, which is what keeps an Agent round trip + /// fast without a heartbeat. + notify: Arc, +} + +/// One relayed agent-bridge answer: the value the MiniApp returned plus the +/// capabilities it last announced. +type AgentAnswer = Result; + +struct PendingAgentRequest { + sender: tokio::sync::oneshot::Sender, + view_id: String, + deadline: std::time::Instant, + claimed: bool, +} + +impl PendingAgentRequest { + fn claim(&mut self, view_id: &str) -> bool { + if self.view_id != view_id + || self.claimed + || self.sender.is_closed() + || std::time::Instant::now() >= self.deadline + { + return false; + } + self.claimed = true; + true + } +} + +/// A Shell advertises what it can do at `open`; an older Shell simply says less. +fn shell_advertises_agent_bridge(params: &Value) -> bool { + params + .get("shellCapabilities") + .and_then(Value::as_array) + .is_some_and(|capabilities| { + capabilities + .iter() + .any(|value| value.as_str() == Some(ipc::MINIAPP_AGENT_BRIDGE_CAPABILITY)) + }) +} + +/// Read one bounded string list the Shell relayed from the document. +/// +/// The document is untrusted input here: names are advisory, so they are +/// trimmed, emptied, and capped instead of being validated against a vocabulary +/// the host does not own. +fn declared_names(answer: &Value, field: &str, limit: usize) -> Vec { + answer + .get(field) + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .take(limit) + .collect() + }) + .unwrap_or_default() } impl ShellLease { @@ -168,6 +495,7 @@ impl ShellLease { self.event_bytes = self.event_bytes.saturating_sub(item.to_string().len()); } } + self.notify.notify_waiters(); } } @@ -370,6 +698,64 @@ async fn open_window( Ok(Ticket { instance, view }) } +/// The desktop app the user published for one MiniApp, if any. +/// +/// Publishing is an explicit user action, and the published bundle is the copy +/// the user double-clicks, so an Agent `open` prefers it over building a second, +/// parallel window. `None` means the app was only ever run in-product. +/// +/// Only macOS has a bundle that can be launched this way; the other platforms +/// keep their menu entry and fall through to the in-product surface. +pub(crate) async fn exported_desktop_app( + app: &tauri::AppHandle, + app_id: &str, +) -> Result, String> { + #[cfg(not(target_os = "macos"))] + { + let _ = (app, app_id); + Ok(None) + } + #[cfg(target_os = "macos")] + { + let channel = app.state::().channel; + let data_dir = app + .state::() + .miniapp_manager + .path_manager() + .user_data_dir(); + let app_id = app_id.to_string(); + let paths = tokio::task::spawn_blocking(move || { + system_install::status(&data_dir, &app_id, channel) + }) + .await + .map_err(|e| e.to_string())??; + Ok(paths.into_iter().next()) + } +} + +/// Launch a published desktop app exactly like a double-click does: the bundle +/// carries its own launch record, so the shell resolves this host from it. +pub(crate) fn launch_desktop_app(bundle: &std::path::Path) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + let mut child = create_command("/usr/bin/open") + .arg("-n") + .arg(bundle) + .spawn() + .map_err(|error| format!("Could not open the published MiniApp app: {error}"))?; + // Reap the launcher without holding a Tokio worker on an independent app. + std::thread::spawn(move || { + let _ = child.wait(); + }); + Ok(()) + } + #[cfg(not(target_os = "macos"))] + { + let _ = bundle; + Err("Published MiniApp apps can only be launched on macOS".into()) + } +} + pub(crate) async fn miniapp_presentation( app: tauri::AppHandle, webview: tauri::Webview, @@ -608,6 +994,7 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result existing.push("focus", Value::Null); return Ok(json!({"reused":true})); } + existing.agent_bridge = shell_advertises_agent_bridge(¶ms); } else { shells.insert( view.view_id.clone(), @@ -618,6 +1005,8 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result events: VecDeque::new(), sequence: 0, event_bytes: 0, + agent_bridge: shell_advertises_agent_bridge(¶ms), + notify: Arc::new(tokio::sync::Notify::new()), }, ); } @@ -644,7 +1033,7 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result .unwrap_or_else(|p| p.into_inner()) .remove(&view.view_id); return Ok( - json!({"presentationCapabilities":[ipc::MINIAPP_CUSTOM_CHROME_CAPABILITY,ipc::MINIAPP_CHAT_SNAPSHOT_CAPABILITY],"transferConfirmed":transfer_confirmed,"instance":instance,"view":view,"app":miniapp,"hostEpoch":host.instances.snapshot().host_epoch,"locale":locale,"appearanceMode":crate::appearance::AppearanceConfig::miniapp_mode(),"ui":ui}), + json!({"presentationCapabilities":[ipc::MINIAPP_CUSTOM_CHROME_CAPABILITY,ipc::MINIAPP_CHAT_SNAPSHOT_CAPABILITY,ipc::MINIAPP_AGENT_BRIDGE_CAPABILITY],"transferConfirmed":transfer_confirmed,"instance":instance,"view":view,"app":miniapp,"hostEpoch":host.instances.snapshot().host_epoch,"locale":locale,"appearanceMode":crate::appearance::AppearanceConfig::miniapp_mode(),"ui":ui}), ); } let instance_id = string(¶ms, "instanceId")?; @@ -681,24 +1070,54 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result } host.instances.touch(&instance_id, &view_id)?; let after = params.get("after").and_then(Value::as_u64).unwrap_or(0); - let mut shells = host.shells.lock().unwrap_or_else(|p| p.into_inner()); - let lease = shells - .get_mut(&view_id) - .ok_or("MiniApp shell lease expired")?; - let gap = lease - .events - .front() - .and_then(|e| e["sequence"].as_u64()) - .is_some_and(|first| first > after + 1); - let events: Vec<_> = lease - .events - .iter() - .filter(|e| e["sequence"].as_u64().unwrap_or(0) > after) - .cloned() - .collect(); - return Ok( - json!({"events":events,"cursor":lease.sequence,"resyncRequired":gap,"pendingView":instance.pending_view}), - ); + // Bounded long-poll: a window with nothing to say holds the call briefly + // instead of answering empty immediately, so a relayed agent request is + // delivered without a heartbeat and without a busy loop. + let wait_ms = params + .get("waitMs") + .and_then(Value::as_u64) + .unwrap_or(0) + .min(2_000); + let mut waited = false; + loop { + let notify = { + let shells = host.shells.lock().unwrap_or_else(|p| p.into_inner()); + shells.get(&view_id).map(|lease| lease.notify.clone()) + }; + let Some(notify) = notify else { + return Ok(json!({"released":true})); + }; + // Register before reading the queue so a push between the two is not + // lost to the wait. + let notified = notify.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + let (gap, cursor, events) = { + let mut shells = host.shells.lock().unwrap_or_else(|p| p.into_inner()); + let lease = shells + .get_mut(&view_id) + .ok_or("MiniApp shell lease expired")?; + let gap = lease + .events + .front() + .and_then(|e| e["sequence"].as_u64()) + .is_some_and(|first| first > after + 1); + let events: Vec<_> = lease + .events + .iter() + .filter(|e| e["sequence"].as_u64().unwrap_or(0) > after) + .cloned() + .collect(); + (gap, lease.sequence, events) + }; + if !events.is_empty() || waited || wait_ms == 0 { + return Ok( + json!({"events":events,"cursor":cursor,"resyncRequired":gap,"pendingView":instance.pending_view}), + ); + } + waited = true; + let _ = tokio::time::timeout(std::time::Duration::from_millis(wait_ms), notified).await; + } } host.instances.touch(&instance_id, &view_id)?; match method { @@ -769,6 +1188,15 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result ) .await } + // The Shell answered a relayed agent-bridge request. Identity was already + // checked against this view's lease, so the answer can only settle a + // request that was pushed to this window. + "agentClaim" => Ok( + json!({"accepted": host.claim_agent_request(&view_id, &string(¶ms, "requestId")?)}), + ), + "agentResponse" => host + .settle_agent_bridge(&view_id, params) + .map(|()| Value::Null), "contentAction" => { Err("MiniApp semantic content actions are not supported by this host".into()) } @@ -780,6 +1208,251 @@ async fn dispatch(app: &tauri::AppHandle, method: &str, params: Value) -> Result mod tests { use super::*; + fn lease(instance_id: &str, agent_bridge: bool) -> ShellLease { + ShellLease { + instance_id: instance_id.to_string(), + nonce: "nonce".to_string(), + streams: HashSet::new(), + events: VecDeque::new(), + sequence: 0, + event_bytes: 0, + agent_bridge, + notify: Arc::new(tokio::sync::Notify::new()), + } + } + + #[test] + fn a_shell_only_advertises_the_bridge_when_it_says_so() { + assert!(shell_advertises_agent_bridge(&json!({ + "shellCapabilities": [ipc::MINIAPP_AGENT_BRIDGE_CAPABILITY] + }))); + // An older Shell sends fewer capabilities, and its absence is the answer. + assert!(!shell_advertises_agent_bridge(&json!({ + "shellCapabilities": [ipc::MINIAPP_CUSTOM_CHROME_CAPABILITY] + }))); + assert!(!shell_advertises_agent_bridge( + &json!({"shellCapabilities": []}) + )); + assert!(!shell_advertises_agent_bridge(&json!({}))); + assert!(!shell_advertises_agent_bridge(&Value::Null)); + } + + #[test] + fn the_agent_bridge_attaches_to_a_ready_window_that_advertised_it() { + let mut host = DesktopMiniApps::new(ipc::HostChannel::Development); + let (instance, view) = host + .instances + .prepare("demo".into(), None, MiniAppViewKind::Window) + .expect("window view"); + host.shells + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(view.view_id.clone(), lease(&instance.instance_id, true)); + + // A prepared view is not the presentation yet: nothing is mounted there. + assert!(host.agent_window("demo").is_none()); + + host.instances + .ready(&instance.instance_id, &view.view_id) + .expect("view becomes primary"); + assert_eq!( + host.agent_window("demo"), + Some(AgentWindow { + view_id: view.view_id.clone(), + instance_id: instance.instance_id.clone(), + }) + ); + assert!(host.agent_window("other").is_none()); + + // A window whose Shell never advertised the capability is not routable: + // relaying into it would only time out. + host.shells + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_mut(&view.view_id) + .expect("lease") + .agent_bridge = false; + assert!(host.agent_window("demo").is_none()); + } + + #[test] + fn an_app_already_on_screen_in_a_tab_is_reported_as_that_presentation() { + let host = DesktopMiniApps::new(ipc::HostChannel::Development); + let (instance, view) = host + .instances + .prepare("demo".into(), None, MiniAppViewKind::Tab) + .expect("tab view"); + // A prepared view is not on screen yet, so it must not block an Agent + // `open` from launching the app's own window. + assert!(host.tab_presentation("demo").is_none()); + + host.instances + .ready(&instance.instance_id, &view.view_id) + .expect("view becomes primary"); + assert_eq!( + host.tab_presentation("demo"), + Some(instance.instance_id.clone()) + ); + assert!(host.tab_presentation("other").is_none()); + // A tab is not a window: nothing here may answer for a window presentation. + assert!(host.agent_window("demo").is_none()); + } + + #[test] + fn answers_for_unknown_or_abandoned_requests_are_refused_not_fatal() { + let host = DesktopMiniApps::new(ipc::HostChannel::Development); + assert!(host + .settle_agent_bridge("view", json!({"requestId": "agent-1", "ok": true})) + .is_err()); + assert!(host + .settle_agent_bridge("view", json!({"ok": true})) + .is_err()); + // A Shell that answers after our timeout finds no waiter and is dropped. + assert!(host + .settle_agent_bridge( + "view", + json!({"requestId": "agent-1", "ok": false, "error": "late"}) + ) + .is_err()); + } + + #[tokio::test] + async fn oversized_window_bridge_answers_fail_the_waiter_and_release_it() { + let host = DesktopMiniApps::new(ipc::HostChannel::Development); + let (sender, receiver) = tokio::sync::oneshot::channel(); + host.agents.lock().unwrap().insert( + "large".into(), + PendingAgentRequest { + sender, + view_id: "view".into(), + deadline: std::time::Instant::now() + std::time::Duration::from_secs(1), + claimed: true, + }, + ); + host.settle_agent_bridge( + "view", + json!({ + "requestId": "large", "ok": true, "value": "界".repeat(12 * 1024) + }), + ) + .unwrap(); + assert!(receiver + .await + .unwrap() + .unwrap_err() + .starts_with("BRIDGE_REJECTED:")); + assert!(host.agents.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn agent_requests_are_claimed_once_by_their_own_view() { + let mut host = DesktopMiniApps::new(ipc::HostChannel::Development); + let window = host.test_window("demo"); + let (answer, ()) = tokio::join!( + host.call_agent_bridge( + "demo", + "act", + Some(json!({"type":"increment"})), + std::time::Duration::from_secs(1) + ), + async { + tokio::task::yield_now().await; + let event = host + .shells + .lock() + .unwrap() + .get(&window.view_id) + .unwrap() + .events + .back() + .unwrap() + .clone(); + let id = event["payload"]["requestId"].as_str().unwrap(); + assert!(event["payload"]["expiresAt"].as_u64().is_some()); + assert!(!host.claim_agent_request("other-view", id)); + assert!(host + .settle_agent_bridge(&window.view_id, json!({"requestId":id,"ok":true})) + .is_err()); + assert!(host.claim_agent_request(&window.view_id, id)); + assert!(!host.claim_agent_request(&window.view_id, id)); + assert!(host + .settle_agent_bridge("other-view", json!({"requestId":id,"ok":true})) + .is_err()); + host.settle_agent_bridge( + &window.view_id, + json!({"requestId":id,"ok":true,"value":1}), + ) + .unwrap(); + // Replaying the retained event after content/WebView reload cannot + // acquire a second execution, including after the answer settled. + assert!(!host.claim_agent_request(&window.view_id, id)); + } + ); + assert_eq!(answer.unwrap().value, json!(1)); + assert!(host.agents.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn expired_and_canceled_requests_cannot_execute_from_the_replay_queue() { + let mut host = DesktopMiniApps::new(ipc::HostChannel::Development); + let window = host.test_window("demo"); + let error = host + .call_agent_bridge("demo", "act", None, std::time::Duration::from_millis(1)) + .await + .unwrap_err(); + assert!(error.starts_with("BRIDGE_TIMEOUT:")); + let event = host + .shells + .lock() + .unwrap() + .get(&window.view_id) + .unwrap() + .events + .back() + .unwrap() + .clone(); + assert!(!host.claim_agent_request( + &window.view_id, + event["payload"]["requestId"].as_str().unwrap() + )); + assert!(host.agents.lock().unwrap().is_empty()); + + // Dropping the tool future closes its receiver even before its deadline. + assert!(tokio::time::timeout( + std::time::Duration::from_millis(1), + host.call_agent_bridge("demo", "act", None, std::time::Duration::from_secs(60)) + ) + .await + .is_err()); + let event = host + .shells + .lock() + .unwrap() + .get(&window.view_id) + .unwrap() + .events + .back() + .unwrap() + .clone(); + assert!(!host.claim_agent_request( + &window.view_id, + event["payload"]["requestId"].as_str().unwrap() + )); + } + + #[test] + fn request_deadlines_are_checked_before_claiming_even_while_the_waiter_is_live() { + let (sender, _receiver) = tokio::sync::oneshot::channel(); + let mut pending = PendingAgentRequest { + sender, + view_id: "view".into(), + deadline: std::time::Instant::now(), + claimed: false, + }; + assert!(!pending.claim("view")); + assert!(!pending.claimed); + } + #[test] fn development_and_installed_hosts_share_data_with_independent_host_leases() { let data = tempfile::tempdir().unwrap(); diff --git a/src/apps/desktop/src/miniapp_web_host.rs b/src/apps/desktop/src/miniapp_web_host.rs new file mode 100644 index 0000000000..ab46ce6f05 --- /dev/null +++ b/src/apps/desktop/src/miniapp_web_host.rs @@ -0,0 +1,741 @@ +//! Desktop adapter for the platform-agnostic MiniApp Web surface port. +//! +//! The Agent drives a MiniApp through BrowserUse-style actions, but the DOM +//! lives inside the product window: only the Web UI can read the frame, walk the +//! document, and dispatch input events. This adapter therefore owns transport +//! only — request correlation, surface readiness, and timeouts — while the +//! product-domain contract, session registry, target handles, and method +//! whitelist stay in `openbitfun_core::agentic::tools::miniapp_surface`. +//! +//! It never lets the Agent reach host paths, Tauri commands, or arbitrary +//! JavaScript: only the whitelisted surface methods cross this boundary. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +use openbitfun_core::agentic::tools::miniapp_surface::{ + set_miniapp_web_host, MiniAppWebCommand, MiniAppWebHost, MiniAppWebOpenRequest, +}; +use openbitfun_core::infrastructure::events::{emit_global_event, BackendEvent}; +use openbitfun_core::miniapp::try_get_global_miniapp_manager; +use openbitfun_product_domains::miniapp::web_surface::{ + MiniAppWebAppSummary, MiniAppWebErrorCode, MiniAppWebSurfaceInfo, + MINIAPP_WEB_BRIDGE_TIMEOUT_MS, MINIAPP_WEB_MAX_APPS, MINIAPP_WEB_MAX_DESCRIPTION_CHARS, +}; +use serde::Deserialize; +use serde_json::{json, Value}; +use tokio::sync::oneshot; + +use crate::miniapp_desktop::{AgentWindow, DesktopMiniApps}; + +const MINIAPP_WEB_REQUEST_EVENT: &str = "agentic://miniapp-web-request"; + +/// Opening may have to create a tab, mount the frame, and wait for the first +/// render, so it is the slowest request on this surface. +const OPEN_TIMEOUT: Duration = Duration::from_secs(90); +/// One DOM action against an already-mounted frame. +const EXECUTE_TIMEOUT: Duration = Duration::from_secs(30); +/// A bridge round trip is bounded by the MiniApp, not by us: give it one DOM +/// action plus the contract timeout and slack, so the surface can report +/// `BRIDGE_TIMEOUT` itself instead of losing the race to our transport. +const BRIDGE_TIMEOUT: Duration = EXECUTE_TIMEOUT.saturating_add(Duration::from_millis( + MINIAPP_WEB_BRIDGE_TIMEOUT_MS + 10_000, +)); +/// A launched desktop app has to start its process, resolve this host, and +/// bootstrap before its window can be attached to. +const WINDOW_BOOT_TIMEOUT: Duration = Duration::from_secs(20); +const WINDOW_BOOT_POLL: Duration = Duration::from_millis(150); + +type PendingResponse = oneshot::Sender>; + +static NEXT_REQUEST_ID: AtomicU64 = AtomicU64::new(1); +static SURFACE_READY: AtomicBool = AtomicBool::new(false); +static PENDING_RESPONSES: OnceLock>> = OnceLock::new(); + +fn pending_responses() -> &'static Mutex> { + PENDING_RESPONSES.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn lock_pending_responses() -> std::sync::MutexGuard<'static, HashMap> { + pending_responses() + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +/// Fail every waiting caller. Used when the surface detaches so a turn never +/// blocks on a listener that is gone. +fn fail_pending(message: &str) { + let pending = std::mem::take(&mut *lock_pending_responses()); + for (_, sender) in pending { + let _ = sender.send(Err(message.to_string())); + } +} + +async fn dispatch_surface_request( + action: &str, + payload: Value, + timeout: Duration, + purpose: &str, +) -> Result { + if !SURFACE_READY.load(Ordering::Acquire) { + return Err(format!( + "{}: the Web UI MiniApp surface listener is not attached", + MiniAppWebErrorCode::SurfaceUnavailable.as_str() + )); + } + let request_id = format!( + "miniapp-web-{}-{}", + std::process::id(), + NEXT_REQUEST_ID.fetch_add(1, Ordering::Relaxed) + ); + let (sender, receiver) = oneshot::channel(); + lock_pending_responses().insert(request_id.clone(), sender); + + let mut payload = match payload { + Value::Object(map) => map, + _ => { + lock_pending_responses().remove(&request_id); + return Err(format!( + "{}: MiniApp surface request payload was not an object", + MiniAppWebErrorCode::InvalidParams.as_str() + )); + } + }; + payload.insert("requestId".to_string(), Value::String(request_id.clone())); + payload.insert("action".to_string(), Value::String(action.to_string())); + + if let Err(error) = emit_global_event(BackendEvent::Custom { + event_name: MINIAPP_WEB_REQUEST_EVENT.to_string(), + payload: Value::Object(payload), + }) + .await + { + lock_pending_responses().remove(&request_id); + return Err(format!( + "{}: failed to deliver the MiniApp surface request: {error}", + MiniAppWebErrorCode::HostError.as_str() + )); + } + + receive_surface_result(&request_id, receiver, timeout, purpose).await +} + +async fn receive_surface_result( + request_id: &str, + receiver: oneshot::Receiver>, + timeout: Duration, + purpose: &str, +) -> Result { + match tokio::time::timeout(timeout, receiver).await { + Ok(Ok(result)) => result, + Ok(Err(_)) => Err(format!( + "{}: MiniApp surface response channel closed", + MiniAppWebErrorCode::HostError.as_str() + )), + Err(_) => { + lock_pending_responses().remove(request_id); + // One slow action does not detach the listener. Readiness follows + // explicit surface lifecycle reports, so a timeout remains retryable. + Err(format!( + "{}: the Web UI MiniApp surface did not {purpose} in time", + MiniAppWebErrorCode::Timeout.as_str() + )) + } + } +} + +/// How one command relates to the app's own window. +/// +/// Three answers, because the interesting failure is not "no window": a session +/// that is bound to an instance the window does not own must not quietly drift to +/// another surface, which is what produced an unrelated `SURFACE_UNAVAILABLE` +/// from the tab runtime. +#[derive(Debug, Clone, PartialEq, Eq)] +enum WindowRoute { + /// The window is the instance this command addresses. + Serve, + /// The app is live in its own window, but the command addresses a different + /// instance: a session from before that window opened, or a retired tab. + Stale(crate::miniapp_desktop::AgentWindow), + /// No window can carry this command, so the tab runtime is what remains. + Absent, +} + +fn window_route( + host: &crate::miniapp_desktop::DesktopMiniApps, + command: &MiniAppWebCommand, +) -> WindowRoute { + let Some(window) = host.agent_window(&command.app_id) else { + return WindowRoute::Absent; + }; + match command.instance_id.as_deref() { + // A command without an instance is a fresh session, which can only be + // talking about the one presentation that exists. + None => WindowRoute::Serve, + Some(bound) if bound == window.instance_id => WindowRoute::Serve, + Some(_) => WindowRoute::Stale(window), + } +} + +/// Answer a surface command through the MiniApp's own window. +/// +/// Bridge observation, semantic actions, and host-routed key input travel this +/// way. A window presentation keeps its pixels and DOM to itself, so other +/// actions are refused with the capability gap instead of being sent somewhere +/// that cannot satisfy them. The envelopes are byte-for-byte the ones the tab +/// runtime returns, so the tool above this adapter has one transport contract. +async fn relay_to_window( + host: &crate::miniapp_desktop::DesktopMiniApps, + command: &MiniAppWebCommand, +) -> Result { + match command.method.as_str() { + "bridge_observe" => { + let answer = host + .call_agent_bridge(&command.app_id, "observe", None, BRIDGE_TIMEOUT) + .await?; + Ok(json!({ + "state": answer.value.get("state").cloned().unwrap_or(Value::Null), + "auto": answer.value.get("auto").cloned().unwrap_or(Value::Null), + "capabilities": answer.capabilities, + "actions": answer.actions, + })) + } + "bridge_act" => { + let action = command + .params + .as_ref() + .and_then(|params| params.get("action")) + .cloned() + .ok_or_else(|| { + format!( + "{}: bridge_act requires an action", + MiniAppWebErrorCode::InvalidParams.as_str() + ) + })?; + let answer = host + .call_agent_bridge(&command.app_id, "act", Some(action), BRIDGE_TIMEOUT) + .await?; + Ok(json!({ + "result": answer.value, + "capabilities": answer.capabilities, + "actions": answer.actions, + })) + } + "key" => { + let params = command.params.clone().unwrap_or_else(|| json!({})); + let key = params + .get("key") + .and_then(Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| { + format!( + "{}: key requires a non-empty key", + MiniAppWebErrorCode::InvalidParams.as_str() + ) + })?; + let answer = host + .call_agent_bridge( + &command.app_id, + "input", + Some(json!({ + "type": "key", + "key": key, + "modifiers": params.get("modifiers").cloned().unwrap_or_else(|| json!([])), + "holdMs": params.get("holdMs").cloned().unwrap_or_else(|| json!(0)), + })), + BRIDGE_TIMEOUT, + ) + .await?; + Ok(json!({ + "result": answer.value, + "capabilities": answer.capabilities, + "actions": answer.actions, + })) + } + method => Err(format!( + "{}: MiniApp {} is running in its own window, which cannot expose DOM structure or pixels, so '{method}' is unavailable there. Its agent bridge (observe_bridge, act_bridge, watch_state) and host-routed key input still work. To use DOM or screenshot actions, move the MiniApp to a tab.", + MiniAppWebErrorCode::CapabilityMissing.as_str(), + command.app_id + )), + } +} + +/// Where an app should appear for this session. +/// +/// The two presentations answer different transports, so the choice is made once, +/// up front, and the surface description is derived from it. +enum Presentation { + /// The app's own window, identified by the instance it registered. + Window(AgentWindow), + /// The in-product tab, which the Web UI owns and mounts. + Tab, +} + +/// The Desktop implementation of the MiniApp Web surface port. +/// +/// Two presentations can host one app, and they are not interchangeable: a tab +/// lives in the product window and can be read, clicked, and captured, while the +/// app's own installed window is a separate process that can only answer the +/// agent bridge it was compiled with. The adapter therefore picks the transport +/// per request — the window when one is live, otherwise the Web UI — and refuses +/// the actions the chosen transport cannot perform instead of pretending. +struct DesktopMiniAppWebHost { + app: tauri::AppHandle, +} + +impl DesktopMiniAppWebHost { + fn miniapps(&self) -> Option> { + use tauri::Manager; + self.app.try_state::() + } + + /// Reject an unknown app before involving the UI, and tell the surface which + /// runtime profile the app is installed with so it can report capabilities + /// honestly instead of guessing from the frame alone. + async fn open_surface( + &self, + request: MiniAppWebOpenRequest, + ) -> Result { + let manager = try_get_global_miniapp_manager().ok_or_else(|| { + format!( + "{}: the MiniApp manager is not running in this process", + MiniAppWebErrorCode::SurfaceUnavailable.as_str() + ) + })?; + let meta = manager + .get_meta(&request.app_id) + .await + .map_err(|error| format!("{}: {error}", MiniAppWebErrorCode::UnknownApp.as_str()))?; + let strict_runtime = manager.uses_market_strict_runtime(&request.app_id).await; + + match self.resolve_presentation(&request.app_id).await? { + Presentation::Window(window) => Ok(MiniAppWebSurfaceInfo::window( + request.app_id, + window.instance_id, + strict_runtime, + )), + // `open_tab_surface` above already attempted this path. Keep this + // branch for a race where another surface mounted the tab between + // the two checks. + Presentation::Tab => { + self.open_tab_surface(&request.app_id, meta.name, strict_runtime) + .await + } + } + } + + /// Choose a fallback presentation after the controllable tab path failed. + /// + /// A tab is attempted by `open_surface` first. This helper only decides what + /// to do when that Web UI route is unavailable: reuse an existing window, + /// launch the published desktop app, or retry a tab as the final fallback. + async fn resolve_presentation(&self, app_id: &str) -> Result { + // A window the user already has open is theirs: attach to it instead of + // moving the app somewhere else. + if let Some(window) = self.miniapps().and_then(|host| host.agent_window(app_id)) { + return Ok(Presentation::Window(window)); + } + + // A tab is part of the product window, so the app is on screen already. + // Opening a second copy of it would only split the user's attention. + if self + .miniapps() + .is_some_and(|host| host.tab_presentation(app_id).is_some()) + { + return Ok(Presentation::Tab); + } + + // An app the user published to the desktop is opened as that app: the + // window it launches is the copy they double-click, and it is the one that + // can report live state to the agent bridge. Failing to launch it is a + // degradation, not an error — the in-product surface still opens. + let Some(bundle) = crate::miniapp_desktop::exported_desktop_app(&self.app, app_id).await? + else { + return Ok(Presentation::Tab); + }; + if let Err(error) = crate::miniapp_desktop::launch_desktop_app(&bundle) { + log::warn!( + "MiniApp {}: the published desktop app could not be launched ({error}); using the in-product surface", + app_id + ); + return Ok(Presentation::Tab); + } + match self.wait_for_window(app_id).await { + Some(window) => Ok(Presentation::Window(window)), + None => { + log::warn!( + "MiniApp {}: the published desktop app did not register a window in time; using the in-product surface", + app_id + ); + Ok(Presentation::Tab) + } + } + } + + /// Ask the Web UI for the in-product surface and adopt the host-owned facts. + async fn open_tab_surface( + &self, + app_id: &str, + name: String, + strict_runtime: bool, + ) -> Result { + let result = dispatch_surface_request( + "open", + json!({ + "appId": app_id, + "name": name, + "strictRuntime": strict_runtime, + }), + OPEN_TIMEOUT, + "open the MiniApp surface", + ) + .await?; + + let mut surface: MiniAppWebSurfaceInfo = + serde_json::from_value(result).map_err(|error| { + format!( + "{}: unreadable surface description: {error}", + MiniAppWebErrorCode::HostError.as_str() + ) + })?; + // The app id and runtime profile are host facts; never let a surface + // response retarget the session to another app. + surface.app_id = app_id.to_string(); + surface.strict_runtime = strict_runtime; + Ok(surface) + } + + /// Wait for a launched desktop app to become an attachable window. + /// + /// The bundle boots its own process and bootstraps against this host, so the + /// presentation exists only after that handshake. Nothing is cached here: the + /// instance the app registers is the one the session must address. + async fn wait_for_window(&self, app_id: &str) -> Option { + let deadline = tokio::time::Instant::now() + WINDOW_BOOT_TIMEOUT; + loop { + if let Some(window) = self.miniapps().and_then(|host| host.agent_window(app_id)) { + return Some(window); + } + if tokio::time::Instant::now() >= deadline { + return None; + } + tokio::time::sleep(WINDOW_BOOT_POLL).await; + } + } +} + +#[async_trait::async_trait] +impl MiniAppWebHost for DesktopMiniAppWebHost { + async fn open(&self, request: MiniAppWebOpenRequest) -> Result { + self.open_surface(request).await + } + + async fn execute(&self, command: MiniAppWebCommand) -> Result { + if let Some(host) = self.miniapps() { + match window_route(&host, &command) { + WindowRoute::Serve => return relay_to_window(&host, &command).await, + // The app is in a window this session never bound to, and the tab + // runtime never mounted it. Answering there would report a surface + // that does not exist and hide the real cause, so name the window + // the caller can reattach to instead. + WindowRoute::Stale(window) => { + return Err(format!( + "{}: MiniApp {} is running in its own window as instance {}, but this session is bound to {}. Call open again to attach to that window.", + MiniAppWebErrorCode::SurfaceUnavailable.as_str(), + command.app_id, + window.instance_id, + command.instance_id.as_deref().unwrap_or("another instance"), + )); + } + WindowRoute::Absent => {} + } + } + let timeout = if command.method.starts_with("bridge_") { + BRIDGE_TIMEOUT + } else { + EXECUTE_TIMEOUT + }; + dispatch_surface_request( + "execute", + json!({ "command": command }), + timeout, + "complete the MiniApp surface action", + ) + .await + } + + /// Discovery reads installed metadata only: no UI round trip, no host paths. + /// A failure stays a failure, because an empty list would tell the Agent that + /// nothing is installed. + async fn list_apps(&self) -> Result, String> { + let manager = try_get_global_miniapp_manager().ok_or_else(|| { + format!( + "{}: the MiniApp manager is not running in this process", + MiniAppWebErrorCode::SurfaceUnavailable.as_str() + ) + })?; + let metas = manager + .list() + .await + .map_err(|error| format!("{}: {error}", MiniAppWebErrorCode::HostError.as_str()))?; + let mut apps = metas + .into_iter() + .map(|meta| { + let strict_runtime = matches!( + meta.runtime_profile, + openbitfun_product_domains::miniapp::types::MiniAppRuntimeProfile::MarketStrict + ); + MiniAppWebAppSummary { + app_id: meta.id, + name: meta.name, + description: meta + .description + .chars() + .take(MINIAPP_WEB_MAX_DESCRIPTION_CHARS) + .collect(), + version: meta.version, + updated_at: meta.updated_at, + strict_runtime, + } + }) + .collect::>(); + // Most recently updated first: the app the user was just working on is + // the one they usually name. + apps.sort_by(|a, b| { + b.updated_at + .cmp(&a.updated_at) + .then_with(|| a.name.cmp(&b.name)) + }); + apps.truncate(MINIAPP_WEB_MAX_APPS); + Ok(apps) + } +} + +/// Install the Desktop surface host. Safe to call once during native setup. +/// +/// The host keeps the app handle only to reach the MiniApp window registry: a +/// MiniApp the user opened from its installed shortcut is presented by another +/// process, and that registry is the one place that knows it exists. +pub(crate) fn install(app: tauri::AppHandle) { + set_miniapp_web_host(std::sync::Arc::new(DesktopMiniAppWebHost { app })); +} + +#[derive(Debug, Default, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub(crate) struct MiniAppWebSurfaceReadyRequest { + ready: bool, +} + +/// Attach or detach the Web UI MiniApp surface listener. +#[tauri::command] +pub(crate) fn mark_miniapp_web_surface_ready(request: Option) { + let ready = request.map(|request| request.ready).unwrap_or(false); + SURFACE_READY.store(ready, Ordering::Release); + if !ready { + fail_pending(&format!( + "{}: the Web UI MiniApp surface detached", + MiniAppWebErrorCode::SurfaceUnavailable.as_str() + )); + } +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ReportMiniAppWebResultRequest { + request_id: String, + success: bool, + #[serde(default)] + result: Option, + #[serde(default)] + error: Option, +} + +/// Return a MiniApp surface result to the waiting Agent call. +#[tauri::command] +pub(crate) async fn report_miniapp_web_result( + request: ReportMiniAppWebResultRequest, +) -> Result<(), String> { + let sender = lock_pending_responses() + .remove(&request.request_id) + .ok_or_else(|| "MiniApp surface request is no longer pending".to_string())?; + let result = if request.success { + Ok(request.result.unwrap_or(Value::Null)) + } else { + Err(request + .error + .filter(|message| !message.trim().is_empty()) + .unwrap_or_else(|| { + format!( + "{}: the MiniApp surface action failed", + MiniAppWebErrorCode::HostError.as_str() + ) + })) + }; + sender + .send(result) + .map_err(|_| "MiniApp surface request receiver is no longer available".to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn an_action_timeout_does_not_detach_the_surface() { + SURFACE_READY.store(true, Ordering::Release); + let (sender, receiver) = oneshot::channel(); + lock_pending_responses().insert("timeout-test".into(), sender); + let result = + receive_surface_result("timeout-test", receiver, Duration::from_millis(1), "answer") + .await; + assert!(result.unwrap_err().starts_with("TIMEOUT:")); + assert!(SURFACE_READY.load(Ordering::Acquire)); + assert!(!lock_pending_responses().contains_key("timeout-test")); + let (sender, receiver) = oneshot::channel(); + sender.send(Ok(json!({"retried": true}))).unwrap(); + assert_eq!( + receive_surface_result("retry-test", receiver, EXECUTE_TIMEOUT, "answer") + .await + .unwrap(), + json!({"retried": true}) + ); + SURFACE_READY.store(false, Ordering::Release); + } + + #[test] + fn readiness_defaults_to_detached_so_calls_fail_loudly() { + let request: MiniAppWebSurfaceReadyRequest = + serde_json::from_value(json!({})).expect("empty request"); + assert!(!request.ready); + let request: MiniAppWebSurfaceReadyRequest = + serde_json::from_value(json!({ "ready": true })).expect("ready request"); + assert!(request.ready); + } + + #[test] + fn bridge_actions_get_a_longer_budget_than_dom_actions() { + let dom = if "inspect".starts_with("bridge_") { + BRIDGE_TIMEOUT + } else { + EXECUTE_TIMEOUT + }; + assert_eq!(dom, EXECUTE_TIMEOUT); + assert!(BRIDGE_TIMEOUT > EXECUTE_TIMEOUT); + } + + #[test] + fn result_reports_accept_camel_case_request_ids() { + let request: ReportMiniAppWebResultRequest = serde_json::from_value(json!({ + "requestId": "miniapp-web-1", + "success": false, + "error": "STALE_TARGET: iframe reloaded", + })) + .expect("report request"); + assert_eq!(request.request_id, "miniapp-web-1"); + assert!(!request.success); + assert!(request.result.is_none()); + } + + #[test] + fn only_commands_bound_to_the_window_are_relayed_to_it() { + let mut host = crate::miniapp_desktop::DesktopMiniApps::new( + openbitfun_miniapp_ipc::HostChannel::Development, + ); + let window = host.test_window("demo"); + + // The session bound by `open` carries the window instance. + assert_eq!( + window_route( + &host, + &MiniAppWebCommand::new("demo", Some(&window.instance_id), "bridge_observe", None) + ), + WindowRoute::Serve + ); + // A command without an instance is a fresh session, which can only be + // talking about the one presentation that exists. + assert_eq!( + window_route( + &host, + &MiniAppWebCommand::new("demo", None, "bridge_observe", None) + ), + WindowRoute::Serve + ); + // Another instance is a different presentation, so this window must not + // answer for it, and the caller has to reattach instead of drifting. + assert!(matches!( + window_route( + &host, + &MiniAppWebCommand::new("demo", Some("instance-other"), "bridge_observe", None) + ), + WindowRoute::Stale(_) + )); + assert_eq!( + window_route( + &host, + &MiniAppWebCommand::new("other", Some(&window.instance_id), "bridge_observe", None) + ), + WindowRoute::Absent + ); + + // A closed window stops being a route, and the tab runtime takes over. + host.instances.detach(&window.instance_id, &window.view_id); + assert_eq!( + window_route( + &host, + &MiniAppWebCommand::new("demo", Some(&window.instance_id), "bridge_observe", None) + ), + WindowRoute::Absent + ); + } + + #[test] + fn the_surface_open_reports_is_the_instance_the_window_answers_for() { + let mut host = crate::miniapp_desktop::DesktopMiniApps::new( + openbitfun_miniapp_ipc::HostChannel::Development, + ); + let window = host.test_window("demo"); + // This is what `open` returns to the tool, and the tool then binds its + // session to the reported id. Reporting the view id here used to bind the + // session to nothing the router recognised, so every bridge request fell + // through to the tab runtime and reported an unrelated surface error. + let surface = + MiniAppWebSurfaceInfo::window("demo".to_string(), window.instance_id.clone(), false); + assert!(surface.is_window_presentation()); + assert!(surface.capabilities.is_empty()); + + let session_instance = surface.instance_id.expect("a window names its instance"); + assert_ne!(session_instance, window.view_id); + assert_eq!( + window_route( + &host, + &MiniAppWebCommand::new("demo", Some(&session_instance), "observe_bridge", None) + ), + WindowRoute::Serve + ); + } + + #[tokio::test] + async fn dom_actions_on_a_window_report_the_capability_gap() { + let host = crate::miniapp_desktop::DesktopMiniApps::new( + openbitfun_miniapp_ipc::HostChannel::Development, + ); + let error = relay_to_window( + &host, + &MiniAppWebCommand::new("demo", None, "inspect", None), + ) + .await + .expect_err("a window cannot serve DOM actions"); + assert!(error.starts_with(MiniAppWebErrorCode::CapabilityMissing.as_str())); + assert!(error.contains("move the MiniApp to a tab"), "{error}"); + + let error = relay_to_window( + &host, + &MiniAppWebCommand::new("demo", None, "bridge_act", Some(json!({}))), + ) + .await + .expect_err("bridge_act needs an action"); + assert!(error.starts_with(MiniAppWebErrorCode::InvalidParams.as_str())); + } +} diff --git a/src/apps/miniapp-shell/AGENTS.md b/src/apps/miniapp-shell/AGENTS.md index 03cdd03ca6..3ebfe0f4bc 100644 --- a/src/apps/miniapp-shell/AGENTS.md +++ b/src/apps/miniapp-shell/AGENTS.md @@ -11,6 +11,11 @@ Build this package in a separate Cargo invocation with the miniapp-shell profile Desktop feature unification or debug builds must never inflate exported apps. The preparation script stages the executable for both raw builds and Tauri's externalBin and enforces a 16 MiB per-architecture artifact regression budget. +`desktop:dev` runs that script once at startup, so a session that hot-reloads the +Desktop host after these sources change keeps launching windows with the previous +shell — a stale shell silently stops advertising capabilities the host relies on. +Re-run `pnpm run prepare:miniapp-shell` after editing this package, and reopen an +open window, before trusting a window-level behavior change. macOS exported bundles copy this executable, not the Desktop host; Windows/Linux shortcuts point to the installed sidecar. No fallback copies the host when a sidecar is missing. The original v1 manifest remains compatible. @@ -39,6 +44,34 @@ The HTML is the same shell used before extraction. Do not copy the Web UI, catalog, backend, or product resources into this package. Keep native heartbeats, permissions, transfer acknowledgements and host-bound identity unchanged. +The Shell also relays the MiniApp agent bridge. It advertises +`miniapp_agent_bridge_v1` at bootstrap, forwards host-pushed `agentRequest` +events into the content frame over the same postMessage protocol a tab uses +(`openbitfun:miniapp-agent-request` / `-response`, paired by `requestId`), and +returns the answer with `agentResponse` plus the capabilities the document last +announced. It never reads the frame itself: DOM traversal, pixel capture, and +synthetic input stay with the Web UI tab runtime, and a window presentation +refuses them rather than approximating them. Polls are bounded long polls with a +short yield, so a relayed request arrives immediately without a busy loop. + +Every Agent request must first acquire a one-time `agentClaim` from its host. +Never execute a rejected, expired or old-document claim, and never settle a +rejected replay on behalf of its original delivery. Invalidate in-flight claims +and pending responses before bootstrapping replacement content. Preserve the +poll cursor for the same host epoch/view and consume reload events before +restarting; a new host/view starts a new cursor. Host and Shell upgrade together +for this protocol. + +A running window must survive a host restart, because the host binds a new +socket directory on every start and the endpoint cached at bootstrap becomes +stale. After three consecutive poll failures the Shell re-reads discovery and +reloads itself once to re-bootstrap; keep that retry bounded with the +`sessionStorage` recovery timestamp so an unavailable host cannot turn into a +reload loop, and skip recovery while a presentation transfer is in flight. A +window that keeps polling a dead endpoint stays visually alive with no lease, +which the host reports as no window at all — the failure looks like "the app is +not open" from the Agent side. + `miniapp-chat.js` is a Shell-local presentation module; the main window's Hi Dock is independent. Only an authorized `chat.claimComposer` reveals the launcher. Keep submissions on `chat:userMessage`/`chat.completeUserMessage`, including apps diff --git a/src/apps/miniapp-shell/src/shell.html b/src/apps/miniapp-shell/src/shell.html index 9eacb166f4..c19feb18ea 100644 --- a/src/apps/miniapp-shell/src/shell.html +++ b/src/apps/miniapp-shell/src/shell.html @@ -48,7 +48,7 @@ const frame = document.getElementById('app'); const status = document.getElementById('status'); const statusText = document.getElementById('status-text'); - let bootstrap, cursor = 0, documentUrl, bridgeGeneration = 0, composerGeneration = 0, bindingGeneration = 0; + let bootstrap, cursor = 0, pollIdentity, documentUrl, bridgeGeneration = 0, composerGeneration = 0, bindingGeneration = 0; const chat = mountMiniAppChat(document.getElementById('miniapp-chat'), { icons, onSend: payload => event('chat:userMessage', payload), @@ -108,6 +108,82 @@ const error = value => { statusText.textContent = String(value); status.hidden = false; document.getElementById('retry').hidden = false; }; const local = (method, params) => native('local', { method, params }); const localMethods = new Set(['dialog.open','dialog.save','dialog.message','system.openExternal','system.revealInFolder','notifications.system']); + + // Agent bridge relay: the host asks for state or runs a declared semantic + // action, and this window forwards it into the frame it is presenting. The + // frame speaks one postMessage protocol, so a window and a tab answer the same + // request the same way; only the relay differs. + const agentRequests = new Map(); + let agentCapabilities = []; + let agentActions = []; + const answerAgent = (requestId, ok, value, error, code) => { + void native('agentResponse', { + requestId, + ok, + value: value ?? null, + error: error ?? null, + code: code ?? null, + capabilities: agentCapabilities, + actions: agentActions, + }).catch(() => {}); + }; + const relayAgentRequest = async request => { + const requestId = String(request?.requestId ?? ''); + if (!requestId || agentRequests.has(requestId)) return; + const generation = bridgeGeneration; + try { + // The host owns replay/cancellation state across WebView reloads. A + // rejected claim must not settle another delivery's live request. + const claim = await native('agentClaim', { requestId }); + if (claim?.accepted !== true) return; + if (generation !== bridgeGeneration || !contentReady || !frame.contentWindow) { + answerAgent(requestId, false, null, 'The MiniApp document is not ready', 'SURFACE_UNAVAILABLE'); + return; + } + if (typeof request.expiresAt === 'number' && Date.now() >= request.expiresAt) { + answerAgent(requestId, false, null, 'The Agent request expired before delivery', 'BRIDGE_TIMEOUT'); + return; + } + } catch (cause) { + answerAgent(requestId, false, null, String(cause), 'SURFACE_UNAVAILABLE'); + return; + } + const timer = setTimeout(() => { + if (!agentRequests.delete(requestId)) return; + answerAgent(requestId, false, null, 'The MiniApp did not answer in time', 'BRIDGE_TIMEOUT'); + }, 15000); + agentRequests.set(requestId, timer); + const payload = { + type: 'openbitfun:miniapp-agent-request', + schemaVersion: request.schemaVersion, + requestId, + method: request.method, + }; + if (request.action !== undefined && request.action !== null) payload.action = request.action; + try { frame.contentWindow.postMessage(payload, '*'); } + catch (cause) { + clearTimeout(timer); agentRequests.delete(requestId); + answerAgent(requestId, false, null, String(cause), 'SURFACE_UNAVAILABLE'); + } + }; + window.addEventListener('message', message => { + if (message.source !== frame.contentWindow || !message.data) return; + if (message.data.type === 'openbitfun:miniapp-agent-ready') { + agentCapabilities = Array.isArray(message.data.capabilities) ? message.data.capabilities.slice(0, 8) : []; + agentActions = Array.isArray(message.data.actions) + ? message.data.actions.filter(name => typeof name === 'string' && name).slice(0, 32) + : []; + return; + } + if (message.data.type !== 'openbitfun:miniapp-agent-response') return; + const requestId = String(message.data.requestId ?? ''); + const timer = agentRequests.get(requestId); + if (!timer) return; + clearTimeout(timer); + agentRequests.delete(requestId); + answerAgent(requestId, message.data.ok === true, message.data.value, message.data.error); + }); + window.addEventListener('message', async message => { if (message.source !== frame.contentWindow || !message.data || typeof message.data.method !== 'string') return; const { id, method, params = {} } = message.data; @@ -147,6 +223,7 @@ } catch (cause) { frame.contentWindow?.postMessage({ jsonrpc: '2.0', id, error: { code: -32000, message: String(cause) } }, '*'); } }); let permissionKey = ''; + let permissionTick = 0; function renderPermissions(requests) { const panel = document.getElementById('permissions'); const key = JSON.stringify(requests); @@ -171,11 +248,30 @@ panel.append(row); } } + // A host restart publishes a new IPC endpoint in `host.json` and forgets every + // lease it used to know, while `poll` keeps talking to the endpoint this window + // resolved at bootstrap. That socket is gone, so retrying can never succeed and + // the window turns into a live-looking app the host cannot see. Rebooting the + // page is the reconnect: `start` resolves the current host and registers again. + function recoverFromHostRestart() { + stopped = true; + const now = Date.now(); + const last = Number(sessionStorage.getItem('miniapp-recover-at') || 0); + // A host that stays unreachable must not turn this into a reload loop, so the + // window shows the error and waits for the user. + if (now - last < 15000) return; + sessionStorage.setItem('miniapp-recover-at', String(now)); + location.reload(); + } async function poll(generation) { + let failures = 0; while (!stopped && generation === pollGeneration) { try { - const batch = await native('poll', { after: cursor }); + const batch = await native('poll', { after: cursor, waitMs: 700 }); if (generation !== pollGeneration) return; + failures = 0; + // The connection is proven, so a later restart may recover again. + sessionStorage.removeItem('miniapp-recover-at'); chat.setConnected(true); if (batch.released) { stopped = true; await native('close'); return; } if (transferring && !batch.pendingView) { transferring = false; titlebar?.setTransferring(false); error(bootstrap.ui.transferFailed); } @@ -188,7 +284,8 @@ for (const item of batch.events ?? []) { if (item.sequence <= cursor) continue; if (item.event === 'focus') await native('focus'); - else if (item.event === 'reload' && !transferring) { void start(); return; } + else if (item.event === 'reload' && !transferring) { cursor = item.sequence; void start(); return; } + else if (item.event === 'agentRequest') void relayAgentRequest(item.payload); else { event(item.event, item.payload); if (item.event === 'agent:event') chat.event(item.payload); @@ -196,14 +293,38 @@ cursor = item.sequence; } cursor = Math.max(cursor, batch.cursor ?? 0); - renderPermissions(await native('permissions')); - } catch (cause) { if (generation !== pollGeneration) return; chat.setConnected(false); error(cause); } - await new Promise(resolve => setTimeout(resolve, 750)); + // An idle poll already takes most of a second, so it can carry the + // permission refresh; a busy one only refreshes periodically. + if ((batch.events ?? []).length === 0 || permissionTick++ % 8 === 0) renderPermissions(await native('permissions')); + } catch (cause) { + if (generation !== pollGeneration) return; + chat.setConnected(false); + error(cause); + // Three failures in a row is a host that is gone, not a slow batch. A + // handover in progress is left alone: it owns its own restart path. + if (++failures >= 3) { + if (transferring) stopped = true; + else recoverFromHostRestart(); + return; + } + } + // Events already arrive promptly through the long poll; this only yields. + await new Promise(resolve => setTimeout(resolve, 50)); } } async function start() { + const generation = ++pollGeneration; + contentReady = false; + bridgeGeneration++; + for (const [requestId, timer] of agentRequests) { + clearTimeout(timer); + answerAgent(requestId, false, null, 'The MiniApp document changed', 'SURFACE_UNAVAILABLE'); + } + agentRequests.clear(); try { - bootstrap = await native('bootstrap'); + const nextBootstrap = await native('bootstrap'); + if (generation !== pollGeneration) return; + bootstrap = nextBootstrap; if (bootstrap.reused) return; const ui = bootstrap.ui; if (bootstrap.instance.primaryView?.kind === 'tab' && !bootstrap.transferConfirmed && !confirm(ui.transferWarning)) { @@ -214,9 +335,15 @@ document.documentElement.dataset.openbitfunAppearanceMode = bootstrap.appearanceMode; contentReady = false; renderTitlebar(); - bridgeGeneration++; composerGeneration++; bindingGeneration++; - cursor = 0; permissionKey = ''; + const nextPollIdentity = JSON.stringify([bootstrap.hostEpoch, bootstrap.view?.viewId]); + if (nextPollIdentity !== pollIdentity) cursor = 0; + pollIdentity = nextPollIdentity; + permissionKey = ''; + // A reloaded document announces its bridge again; holding the previous + // declaration would report a capability the new document may not have. + agentCapabilities = []; + agentActions = []; const locale = bootstrap.locale; chat.configure({ labels: chatLabels[locale] || chatLabels[windowLabels.aliases[locale.toLowerCase()] || windowLabels.fallbackLocale], @@ -232,14 +359,14 @@ if (documentUrl) URL.revokeObjectURL(documentUrl); documentUrl = URL.createObjectURL(new Blob([bootstrap.app.compiled_html], { type: 'text/html;charset=utf-8' })); frame.onload = async () => { - try { await native('ready'); contentReady = true; titlebar?.setReady(true); status.hidden = true; frame.hidden = false; event('activate', {}); } + try { if (generation !== pollGeneration) return; await native('ready'); if (generation !== pollGeneration) return; contentReady = true; titlebar?.setReady(true); status.hidden = true; frame.hidden = false; event('activate', {}); } catch (cause) { error(cause); } }; frame.setAttribute('sandbox', bootstrap.app.runtime_profile === 'market_strict' ? 'allow-scripts allow-downloads' : 'allow-scripts allow-forms allow-modals allow-popups allow-downloads'); frame.src = documentUrl; stopped = false; - void poll(++pollGeneration); - } catch (cause) { error(cause); await native('focus'); } + void poll(generation); + } catch (cause) { if (generation !== pollGeneration) return; error(cause); await native('focus'); } } document.getElementById('retry').onclick = start; renderTitlebar(); diff --git a/src/apps/miniapp-shell/src/shell.rs b/src/apps/miniapp-shell/src/shell.rs index 87ed87ad9a..628b5cc68d 100644 --- a/src/apps/miniapp-shell/src/shell.rs +++ b/src/apps/miniapp-shell/src/shell.rs @@ -162,7 +162,7 @@ pub(crate) async fn miniapp_shell_call( let value = ipc::call( &endpoint, "open", - json!({"launchId":state.record.launch_id,"nonce":state.nonce,"shellCapabilities":[ipc::MINIAPP_CUSTOM_CHROME_CAPABILITY,ipc::MINIAPP_CHAT_SNAPSHOT_CAPABILITY]}), + json!({"launchId":state.record.launch_id,"nonce":state.nonce,"shellCapabilities":[ipc::MINIAPP_CUSTOM_CHROME_CAPABILITY,ipc::MINIAPP_CHAT_SNAPSHOT_CAPABILITY,ipc::MINIAPP_AGENT_BRIDGE_CAPABILITY]}), ) .await?; if value["reused"] == true { @@ -189,9 +189,8 @@ pub(crate) async fn miniapp_shell_call( window.show().map_err(|e| e.to_string())?; Ok(value) } - "ready" | "tab" | "poll" | "permissions" | "chatSnapshot" => { - host_call(&state, &request.operation, params).await - } + "ready" | "tab" | "poll" | "permissions" | "chatSnapshot" | "agentResponse" + | "agentClaim" => host_call(&state, &request.operation, params).await, "close" => { let _ = tokio::time::timeout( std::time::Duration::from_secs(1), diff --git a/src/crates/adapters/miniapp-ipc/src/lib.rs b/src/crates/adapters/miniapp-ipc/src/lib.rs index 4bfeb48938..3fe9c60b7b 100644 --- a/src/crates/adapters/miniapp-ipc/src/lib.rs +++ b/src/crates/adapters/miniapp-ipc/src/lib.rs @@ -9,6 +9,12 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; pub const MINIAPP_CUSTOM_CHROME_CAPABILITY: &str = "miniapp_custom_chrome_v1"; pub const MINIAPP_CHAT_SNAPSHOT_CAPABILITY: &str = "miniapp_chat_snapshot_v1"; +/// The Shell can relay the Agent's bridge requests into the MiniApp frame it is +/// presenting. Advertised at `open` so an older Shell never receives a request it +/// cannot answer, and so the host only routes to windows that can take them. +pub const MINIAPP_AGENT_BRIDGE_CAPABILITY: &str = "miniapp_agent_bridge_v1"; +/// One relayed request or answer, bounded like every other host frame. +pub const MINIAPP_AGENT_MESSAGE_MAX_BYTES: usize = 32 * 1024; const MAX_FRAME: usize = 16 * 1024 * 1024; diff --git a/src/crates/assembly/core/builtin_skills/miniapp-dev/api-reference.md b/src/crates/assembly/core/builtin_skills/miniapp-dev/api-reference.md index 61e8be39c4..3e09b9ade7 100644 --- a/src/crates/assembly/core/builtin_skills/miniapp-dev/api-reference.md +++ b/src/crates/assembly/core/builtin_skills/miniapp-dev/api-reference.md @@ -13,7 +13,7 @@ MiniApp **能且只能**用以下 API，没有任何"通用 OpenBitFun 后端通 - `app.net.fetch` —— HTTP 请求（受 `permissions.net.allow` 域名白名单限制） - `app.os.info` —— 只读系统信息 - `app.storage.get/set` —— 每应用独立 KV 存储 -- `app.agent.*` —— 小应用自有隐藏 Agent 会话；支持有界只读上下文快照 +- `app.agent.*` —— 小应用自有隐藏 Agent 会话；支持有界只读上下文快照；`app.agent.expose()` 可选地把实时状态/语义动作（含可选的动作名清单 `actions`）暴露给宿主 Agent（不接入也自动上报画布文字与帧存活） - `app.ai.complete / chat / cancel / getModels` —— 复用宿主 AI（无需 API Key） - `app.dialog.open/save/message` —— 文件对话框 - `app.clipboard.readText/writeText` —— 剪贴板 @@ -187,6 +187,54 @@ await app.agent.run('分析当前盘面。', { 需要当前新闻、公告或财务依据时，Agent 仍可独立使用 `WebSearch` / `WebFetch`，并区分数据时间、 发布时间与推断。不要用“只允许引用快照字段”的指令关闭这些研究能力。 +### `app.agent.expose()` — 把小应用自身的实时状态暴露给 Agent + +方向与 `app.agent.run()` 相反：宿主 Agent 来驱动这个小应用。注册后，Agent 端 `MiniAppBrowser` +的 `observe_bridge` 会拿到 `observe()` 的返回值，`act_bridge` 会调用 `act(action)`。 + +这是**可选增强**，不是接入前提：注入到每个小应用页面的运行时会自动上报一层观测（canvas 上的文字、 +动画帧是否在跑、按键/点击是否真的到达），所以未做任何改动的小应用也能被 Agent 观测与操作， +只是那层观测没有语义标签（只能看到画出来的字，读不到「生命值」这种命名状态）。 + +```javascript +let score = 0; +let wave = 1; + +app.agent.expose({ + // 只暴露你愿意让 Agent 看到的字段；返回必须是可 JSON 序列化的有界对象。 + observe: () => ({ + score, + wave, + lives: 3, + player: { x: player.x, y: player.y }, + }), + // 可选：列出 act 认得的动作名（就是 action.type）。不写即旧行为，Agent 会试一个再看结果。 + actions: ['restart', 'move', 'autoplay'], + // 语义动作由小应用自己定义：怎么理解、允不允许，都归你判断。 + act: (action) => { + if (action?.type === 'restart') { score = 0; wave = 1; return { restarted: true }; } + if (action?.type === 'move' && typeof action.dx === 'number') { player.x += action.dx; return { x: player.x }; } + return { ignored: true }; + }, +}); +``` + +要点： + +- 无需要求 `permissions.agent.*`：数据只在页面内流转，不触碰宿主能力。 +- 每个渲染帧都调用 `expose()` 只会替换处理器，不会重复注册；重启/换关后重新 `expose()` 即可对齐新状态。 +- `observe` 抛出异常会被宿主收到为可读的失败，不会挂死请求；长时间阻塞的处理会触发宿主超时。 +- 对纯 canvas、游戏或其他自定义渲染界面，自动观测能给到画出来的文字、动画循环是否在跑、输入是否到达； + 但命名状态和语义动作需要 `expose()`。如果任务依赖连续变化的状态，应暴露完成任务所需的少量权威字段与动作， + 避免 Agent 只能靠逐帧截图或反复试探。 +- 只返回必要的少量字段（分数、生命、关卡、关键坐标等），不要把整帧像素或大数组塞进 `observe`。 +- 对可预测的业务流程，动作可以对应完整且有界的用户意图（例如打开记录、应用筛选、提交表单）， + 让 Agent 能直接执行已知步骤，再用 `observe()` 确认结果；不要强迫它在每个低层输入后都重新观察。 +- `actions` 是可选的动作词表：写了，Agent 在 `observe_bridge` 里就能看到「这个应用认哪些动作」， + 不必猜名字或翻源码；没写，Agent 会先发一个动作再读返回值。词表只是提示，不是白名单， + 在 `act` 里不认的动作照样由你自己处理（例如返回 `{ ignored: true }`）。最多 32 个， + 每个名字 trim 后 1-48 个可打印 ASCII 字符，区分大小写；接受 `['restart']` 或 `[{ type: 'restart' }]` 两种写法。 + ### `app.dialog.*` — 系统对话框 ```javascript diff --git a/src/crates/assembly/core/src/agentic/agents/definitions/modes/creative.rs b/src/crates/assembly/core/src/agentic/agents/definitions/modes/creative.rs index 8175e62da7..b81a94209f 100644 --- a/src/crates/assembly/core/src/agentic/agents/definitions/modes/creative.rs +++ b/src/crates/assembly/core/src/agentic/agents/definitions/modes/creative.rs @@ -90,6 +90,15 @@ mod tests { } } + #[test] + fn creative_mode_can_operate_the_app_it_built() { + // Building a MiniApp without being able to open and use it leaves the + // Agent driving a copy in a headless browser instead of the app the + // user is watching. + let tools = CreativeHarness::new().default_tools(); + assert!(tools.contains(&"MiniAppBrowser".to_string())); + } + #[test] fn creative_prompt_has_its_own_persistent_cache_identity() { let mode = CreativeHarness::new(); diff --git a/src/crates/assembly/core/src/agentic/agents/mod.rs b/src/crates/assembly/core/src/agentic/agents/mod.rs index 057bcaa9ca..9c4c005a6f 100644 --- a/src/crates/assembly/core/src/agentic/agents/mod.rs +++ b/src/crates/assembly/core/src/agentic/agents/mod.rs @@ -118,6 +118,11 @@ pub fn standard_harness_tools() -> Vec { "ReviewPlatform".to_string(), "OpenBitFunControl".to_string(), "ControlHub".to_string(), + // A MiniApp is a Web page this product hosts, so operating one belongs + // to the same surface family as ControlHub: the Agent opens the app in + // the window the user is watching instead of driving a copy elsewhere. + // The tool stays deferred and degrades honestly when no surface exists. + "MiniAppBrowser".to_string(), // Pairs with ControlHub: its `wait` sends anything repeating, or // further out than an hour, to Cron rather than holding the turn open // for the interval. diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs index 3af34ced64..c2d406f951 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/control_hub_tool.rs @@ -2922,16 +2922,7 @@ Results: `{ok, domain, action, data? , error?: {code, message, hints}}`. Check o // model never sees just invites endless retries with // different params. Soft-degrade inside the envelope // rather than erroring, so the turn is not broken. - let text_only = !context.primary_model_supports_image_understanding(); - let no_multimodal_tool_output = !context - .primary_model_facts() - .multimodal_tool_output_supported(); - if text_only || no_multimodal_tool_output { - let reason = if text_only { - "primary_model_is_text_only" - } else { - "provider_no_multimodal_tool_output" - }; + if let Some(reason) = context.image_observation_unavailable_reason() { return Ok(vec![ToolResult::ok( json!({ "success": false, diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/actions.rs b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/actions.rs new file mode 100644 index 0000000000..20ec3926ab --- /dev/null +++ b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/actions.rs @@ -0,0 +1,1506 @@ +//! Action semantics for the MiniAppBrowser tool. +//! +//! This layer decides what each Agent action means: which surface method it +//! maps to, which parameters it needs, and which capability it requires. It +//! holds no transport details (that is the host port) and no envelope shapes +//! (that is [`super::result`]). + +use crate::agentic::tools::framework::ToolResult; +use crate::agentic::tools::miniapp_surface::{ + classify_surface_error, close_session, execute_surface, list_installed_apps, miniapp_web_host, + open_session, parse_find_payload, parse_inspect_payload, refresh_bridge_actions, + refresh_bridge_capabilities, resolve_target, sessions_snapshot, sessions_snapshot_all, + MiniAppWebCommand, MiniAppWebOpenRequest, MiniAppWebSessionSnapshot, +}; +use openbitfun_product_domains::miniapp::web_surface::{ + match_installed_app, validate_bridge_payload_size, MiniAppWebAppMatch, MiniAppWebAppSummary, + MiniAppWebCapture, MiniAppWebErrorCode, MiniAppWebLocator, MINIAPP_WEB_DEFAULT_CAPTURE_EDGE, + MINIAPP_WEB_DEFAULT_WAIT_MS, MINIAPP_WEB_DEFAULT_WATCH_MS, MINIAPP_WEB_DEFAULT_WATCH_POLL_MS, + MINIAPP_WEB_MAX_CAPTURE_EDGE, MINIAPP_WEB_MAX_DEPTH, MINIAPP_WEB_MAX_KEY_HOLD_MS, + MINIAPP_WEB_MAX_NODES, MINIAPP_WEB_MAX_WAIT_MS, MINIAPP_WEB_MAX_WATCH_MS, + MINIAPP_WEB_MAX_WATCH_SAMPLES, MINIAPP_WEB_MAX_WATCH_SAMPLE_BYTES, + MINIAPP_WEB_MIN_WATCH_POLL_MS, +}; +use serde_json::{json, Value}; +use std::time::Duration; + +use super::result::{ + err_response, missing_param, ok_response, ok_response_with_image, render_nodes, surface_error, + target_error, +}; + +/// Every action the tool accepts. Parsing lives here so the schema, the +/// dispatcher, and input validation cannot drift apart. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Action { + Open, + Close, + Sessions, + /// Installed-app discovery: what exists and how it is addressed. + Apps, + Inspect, + Find, + Read, + Click, + Type, + Key, + Scroll, + Wait, + Screenshot, + ObserveBridge, + ActBridge, + /// Bounded sampling window over the bridge: one call instead of a turn-paced + /// observe/wait loop, which is what makes following a running app practical. + WatchState, +} + +pub(super) const ACTIONS: &[&str] = &[ + "open", + "close", + "sessions", + "apps", + "inspect", + "find", + "read", + "click", + "type", + "key", + "scroll", + "wait", + "screenshot", + "observe_bridge", + "act_bridge", + "watch_state", +]; + +impl Action { + pub(super) fn parse(value: &str) -> Option { + match value.trim() { + "open" => Some(Self::Open), + "close" => Some(Self::Close), + "sessions" => Some(Self::Sessions), + "apps" => Some(Self::Apps), + "inspect" => Some(Self::Inspect), + "find" => Some(Self::Find), + "read" => Some(Self::Read), + "click" => Some(Self::Click), + "type" => Some(Self::Type), + "key" => Some(Self::Key), + "scroll" => Some(Self::Scroll), + "wait" => Some(Self::Wait), + "screenshot" => Some(Self::Screenshot), + "observe_bridge" => Some(Self::ObserveBridge), + "act_bridge" => Some(Self::ActBridge), + "watch_state" => Some(Self::WatchState), + _ => None, + } + } + + pub(super) fn as_str(self) -> &'static str { + match self { + Self::Open => "open", + Self::Close => "close", + Self::Sessions => "sessions", + Self::Apps => "apps", + Self::Inspect => "inspect", + Self::Find => "find", + Self::Read => "read", + Self::Click => "click", + Self::Type => "type", + Self::Key => "key", + Self::Scroll => "scroll", + Self::Wait => "wait", + Self::Screenshot => "screenshot", + Self::ObserveBridge => "observe_bridge", + Self::ActBridge => "act_bridge", + Self::WatchState => "watch_state", + } + } + + /// Actions that read or write the MiniApp document need a readable frame. + /// A key is deliberately document-independent: Canvas games and independent + /// windows can receive it through their authenticated Shell relay. + pub(super) fn requires_dom(self) -> bool { + matches!( + self, + Self::Inspect + | Self::Find + | Self::Read + | Self::Click + | Self::Type + | Self::Scroll + | Self::Screenshot + ) + } +} + +pub(super) fn action_from_input(input: &Value) -> Option { + input + .get("action") + .and_then(Value::as_str) + .and_then(Action::parse) +} + +pub(super) async fn dispatch(action: Action, input: &Value) -> Vec { + match action { + Action::Sessions => list_sessions().await, + Action::Apps => installed_apps().await, + Action::Open => open(input).await, + Action::Close => match session_for(input).await { + Ok(session) => close(&session).await, + Err(response) => response, + }, + action => { + let session = match session_for(input).await { + Ok(session) => session, + Err(response) => return response, + }; + if let Some(response) = dom_gate(action, &session) { + return response; + } + match action { + Action::Inspect => inspect(&session, input).await, + Action::Find => find(&session, input).await, + Action::Read => read(&session, input).await, + Action::Click => click(&session, input).await, + Action::Type => type_text(&session, input).await, + Action::Key => press_key(&session, input).await, + Action::Scroll => scroll(&session, input).await, + Action::Wait => wait(&session, input).await, + Action::Screenshot => screenshot(&session, input).await, + Action::ObserveBridge => observe_bridge(&session).await, + Action::ActBridge => act_bridge(&session, input).await, + Action::WatchState => watch_state(&session, input).await, + Action::Open | Action::Close | Action::Sessions | Action::Apps => { + unreachable!("handled before session resolution") + } + } + } + } +} + +async fn session_for(input: &Value) -> Result> { + let Some(session_id) = read_string(input, "sessionId") else { + return Err(missing_param( + input.get("action").and_then(Value::as_str).unwrap_or("?"), + "sessionId", + )); + }; + sessions_snapshot(&session_id).await.ok_or_else(|| { + err_response( + "session", + MiniAppWebErrorCode::UnknownSession, + format!("No MiniApp Web session '{session_id}'."), + &["Open the MiniApp first, then reuse the returned sessionId."], + ) + }) +} + +fn dom_gate(action: Action, session: &MiniAppWebSessionSnapshot) -> Option> { + if !action.requires_dom() || session.has_dom() { + return None; + } + // Two different gaps look identical to a caller that only checks `capabilities`: + // an app that has no readable document, and an app presented in its own window + // that cannot expose DOM or pixels. Naming the right one keeps the + // model from asking the app author for a bridge that already exists. + let (message, hints): (&str, &[&str]) = match session.is_window_presentation() { + true => ( + "This MiniApp runs in its own window, which cannot expose DOM structure or pixels.", + &[ + "observe_bridge, watch_state, act_bridge, and key still reach that window.", + "For page structure, clicks, typing, or screenshots, ask the user to move the MiniApp to a tab.", + ], + ), + false => ( + "This MiniApp surface exposes no readable DOM, so page structure and element targets are unavailable.", + &[ + "Report the capability gap instead of guessing internal state.", + "A Canvas or game MiniApp can declare the optional agent bridge (observe/act) to expose bounded JSON state.", + ], + ), + }; + Some(err_response( + action.as_str(), + MiniAppWebErrorCode::CapabilityMissing, + message, + hints, + )) +} + +/// Forward one request through the host port, turning a typed failure into a +/// model-facing error. Every action goes through this single seam. +async fn surface_call( + action: Action, + session: &MiniAppWebSessionSnapshot, + method: &str, + params: Option, +) -> Result> { + execute_surface(MiniAppWebCommand::new( + &session.app_id, + session.instance_id.as_deref(), + method, + params, + )) + .await + .map_err(|(_code, message)| surface_error(action.as_str(), message)) +} + +/// Resolve a required `targetId` into the surface-internal node key. +async fn node_key( + action: Action, + session: &MiniAppWebSessionSnapshot, + input: &Value, +) -> Result<(String, String), Vec> { + let Some(target_id) = read_string(input, "targetId") else { + return Err(missing_param(action.as_str(), "targetId")); + }; + let node_key = resolve_target(&session.session_id, &target_id) + .await + .map(|(node_key, _summary)| node_key) + .map_err(|code| target_error(action.as_str(), code))?; + Ok((target_id, node_key)) +} + +async fn list_sessions() -> Vec { + let sessions = sessions_snapshot_all().await; + let summary = if sessions.is_empty() { + "No MiniApp Web sessions are open.".to_string() + } else { + sessions + .iter() + .map(|session| { + format!( + "{} appId={} capabilities=[{}]", + session.session_id, + session.app_id, + session.capabilities.join(",") + ) + }) + .collect::>() + .join("\n") + }; + ok_response( + Action::Sessions.as_str(), + json!({ + "sessions": sessions.iter().map(MiniAppWebSessionSnapshot::to_value).collect::>(), + }), + summary, + ) +} + +/// Installed-app discovery. +/// +/// The Agent addresses an app by id, but people name apps. Without discovery the +/// model either guesses ids or reaches for host paths, so this lists what the +/// host can present with the id that addresses it. +async fn installed_apps() -> Vec { + let action = Action::Apps.as_str(); + let apps = match list_installed_apps().await { + Ok(apps) => apps, + Err((code, message)) => { + return err_response( + action, + code, + message, + &["Controlling a MiniApp requires an active Desktop product surface."], + ) + } + }; + let summary = if apps.is_empty() { + "No MiniApps are installed in this profile.".to_string() + } else { + apps.iter() + .map(|app| { + let strict = if app.strict_runtime { + " [strict runtime: bridge only]" + } else { + "" + }; + format!("{} | {} (v{}){}", app.app_id, app.name, app.version, strict) + }) + .collect::>() + .join("\n") + }; + ok_response( + action, + json!({ + "apps": apps.iter().map(MiniAppWebAppSummary::to_value).collect::>(), + }), + summary, + ) +} + +/// Resolve `appId` or `name` into one installed app id. +/// +/// Resolution stays here instead of the model reading host metadata: an exact id +/// or name wins, a unique substring resolves, and anything ambiguous is reported +/// with candidates rather than guessed. +async fn resolve_app_reference(action: &str, input: &Value) -> Result> { + let Some(requested) = read_string(input, "appId").or_else(|| read_string(input, "name")) else { + return Err(missing_param(action, "appId")); + }; + let apps = match list_installed_apps().await { + Ok(apps) => apps, + // A host that cannot enumerate apps still accepts an explicit reference; + // `open` then reports UNKNOWN_APP itself instead of us inventing a match. + Err(_) => return Ok(requested), + }; + match match_installed_app(&apps, &requested) { + MiniAppWebAppMatch::Single(app_id) => Ok(app_id), + MiniAppWebAppMatch::Ambiguous(ids) => Err(err_response( + action, + MiniAppWebErrorCode::InvalidParams, + format!( + "'{requested}' matches {} MiniApps: {}.", + ids.len(), + ids.join(", ") + ), + &["Pass appId to choose one of those MiniApps."], + )), + MiniAppWebAppMatch::Missing => { + let hints = available_app_hints(&apps); + let hints = hints.iter().map(String::as_str).collect::>(); + Err(err_response( + action, + MiniAppWebErrorCode::UnknownApp, + format!("No installed MiniApp matches '{requested}'."), + &hints, + )) + } + } +} + +fn available_app_hints(apps: &[MiniAppWebAppSummary]) -> Vec { + if apps.is_empty() { + return vec!["No MiniApps are installed in this profile.".to_string()]; + } + let listed = apps + .iter() + .take(10) + .map(|app| format!("{} ({})", app.name, app.app_id)) + .collect::>(); + let mut hint = format!("Installed MiniApps: {}.", listed.join(", ")); + if apps.len() > 10 { + hint.push_str(&format!( + " {} more; call apps for the full list.", + apps.len() - 10 + )); + } + vec![hint] +} + +async fn open(input: &Value) -> Vec { + let action = Action::Open.as_str(); + let app_id = match resolve_app_reference(action, input).await { + Ok(app_id) => app_id, + Err(response) => return response, + }; + let host = match miniapp_web_host() { + Ok(host) => host, + Err(error) => { + return err_response( + action, + MiniAppWebErrorCode::SurfaceUnavailable, + error, + &["Controlling a MiniApp requires an active Desktop product surface."], + ) + } + }; + let surface = match host + .open(MiniAppWebOpenRequest { + app_id: app_id.clone(), + }) + .await + { + Ok(surface) => surface, + Err(error) => { + let code = classify_surface_error(&error); + return err_response(action, code, error, &[]); + } + }; + let session = open_session(surface).await; + + let mut summary = format!( + "Opened MiniApp '{}' ({}). capabilities=[{}]", + session.app_id, + session.session_id, + session.capabilities.join(",") + ); + if session.bridge_announced() { + summary.push_str(&format!( + "\nAgent bridge capabilities=[{}]", + session.bridge_capabilities.join(",") + )); + let declared_state = session.has_bridge_capability("observe"); + let declared_actions = session.has_bridge_capability("act"); + if !declared_state { + summary.push_str( + "\nobserve_bridge reports automatic observation only (drawn text, animation liveness, input echo). It still needs the app to declare state before it can report scores or lives by name.", + ); + } + if !declared_actions { + summary.push_str( + "\nThe app declared no semantic actions: drive it with key, click, and drag.", + ); + } + } else if session.has_dom() { + summary.push_str("\nNo agent bridge announced. Continue with inspect."); + } + if session.strict_runtime && !session.has_dom() { + summary.push_str( + "\nThis MiniApp runs in the strict runtime: only bridge actions can be observed.", + ); + } + if session.presented_from.as_deref() == Some("window") { + summary.push_str( + "\nIt was open in its own window; it now runs in the tab the user is watching, so you can read and drive the same instance.", + ); + } + if session.is_window_presentation() { + // Say which transport was attached instead of letting the model discover + // it through a failing action: the window is the instance the user is + // watching, and it answers only what the app itself exposes. + summary.push_str( + "\nYou are attached to the MiniApp's own window, the instance the user has on screen. observe_bridge, watch_state, act_bridge, and key reach it; inspect, click, type, scroll, and screenshot need a tab and are refused here.", + ); + } + ok_response(action, session.to_value(), summary) +} + +async fn close(session: &MiniAppWebSessionSnapshot) -> Vec { + let action = Action::Close.as_str(); + let released = close_session(&session.session_id).await; + ok_response( + action, + json!({ "sessionId": session.session_id, "released": released }), + format!( + "Released MiniApp Web session {}. The app stays open for the user.", + session.session_id + ), + ) +} + +async fn inspect(session: &MiniAppWebSessionSnapshot, _input: &Value) -> Vec { + let action = Action::Inspect.as_str(); + let payload = match surface_call( + Action::Inspect, + session, + "inspect", + Some(json!({ + "maxNodes": MINIAPP_WEB_MAX_NODES, + "maxDepth": MINIAPP_WEB_MAX_DEPTH, + })), + ) + .await + { + Ok(payload) => payload, + Err(response) => return response, + }; + + let (url, title, tree, node_count, truncated) = match parse_inspect_payload(&payload) { + Ok(parsed) => parsed, + Err(_) => { + return err_response( + action, + MiniAppWebErrorCode::HostError, + "The surface returned an unreadable page structure.", + &["Re-run inspect; if it repeats, the MiniApp frame is not ready."], + ) + } + }; + + let rendered = render_nodes(&session.session_id, tree).await; + let mut summary = format!( + "{} ({}) nodes={} capabilities=[{}]", + title.as_deref().unwrap_or("MiniApp"), + url.as_deref().unwrap_or(&session.app_id), + node_count, + session.capabilities.join(",") + ); + if truncated { + summary.push_str("\nStructure was truncated at the observation limits."); + } + if rendered.lines.is_empty() { + summary.push_str("\nNo actionable nodes were exposed."); + } else { + summary.push('\n'); + summary.push_str(&rendered.lines.join("\n")); + } + + ok_response( + action, + json!({ + "sessionId": session.session_id, + "url": url, + "title": title, + "nodeCount": node_count, + "truncated": truncated, + "tree": rendered.tree, + }), + summary, + ) +} + +async fn find(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Find.as_str(); + let locator = input + .get("locator") + .cloned() + .and_then(|value| serde_json::from_value::(value).ok()) + .unwrap_or_default() + .sanitized(); + if locator.is_empty() { + return err_response( + action, + MiniAppWebErrorCode::InvalidParams, + "find requires a locator with at least one of testId, role, name, text, or selector.", + &[], + ); + } + + let mut warnings = Vec::new(); + if locator.is_selector_fallback() { + warnings.push( + "A CSS selector is a compatibility fallback; prefer testId or an accessible name when one exists.", + ); + } + + let payload = match surface_call( + Action::Find, + session, + "find", + Some(json!({ "locator": locator.to_value() })), + ) + .await + { + Ok(payload) => payload, + Err(response) => return response, + }; + + let rendered = render_nodes(&session.session_id, parse_find_payload(&payload)).await; + let count = rendered + .tree + .as_array() + .map(|matches| matches.len()) + .unwrap_or(0); + + let mut summary = if count == 0 { + "No node matched the locator.".to_string() + } else { + rendered.lines.join("\n") + }; + for warning in &warnings { + summary.push_str("\n- "); + summary.push_str(warning); + } + + ok_response( + action, + json!({ + "sessionId": session.session_id, + "matches": rendered.tree, + "warnings": warnings, + }), + summary, + ) +} + +async fn read(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Read.as_str(); + let (target_id, node_key) = match node_key(Action::Read, session, input).await { + Ok(value) => value, + Err(response) => return response, + }; + let payload = match surface_call( + Action::Read, + session, + "read", + Some(json!({ + "nodeKey": node_key, + "attribute": read_string(input, "attribute"), + })), + ) + .await + { + Ok(payload) => payload, + Err(response) => return response, + }; + + let text = payload.get("text").cloned().unwrap_or(Value::Null); + let value = payload.get("value").cloned().unwrap_or(Value::Null); + let summary = format!( + "text={} value={}", + text.as_str().unwrap_or("null"), + value.as_str().unwrap_or("null") + ); + ok_response( + action, + json!({ + "sessionId": session.session_id, + "targetId": target_id, + "text": text, + "value": value, + "attributes": payload.get("attributes").cloned().unwrap_or_else(|| json!({})), + }), + summary, + ) +} + +async fn click(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Click.as_str(); + let (target_id, node_key) = match node_key(Action::Click, session, input).await { + Ok(value) => value, + Err(response) => return response, + }; + match surface_call( + Action::Click, + session, + "click", + Some(json!({ "nodeKey": node_key })), + ) + .await + { + Ok(payload) => ok_response( + action, + json!({ "sessionId": session.session_id, "targetId": target_id, "result": payload }), + format!( + "Clicked {target_id}. Inspect or wait then inspect to confirm the page updated." + ), + ), + Err(response) => response, + } +} + +async fn type_text(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Type.as_str(); + let Some(text) = input.get("text").and_then(Value::as_str) else { + return missing_param(action, "text"); + }; + let (target_id, node_key) = match node_key(Action::Type, session, input).await { + Ok(value) => value, + Err(response) => return response, + }; + let clear = input.get("clear").and_then(Value::as_bool).unwrap_or(true); + match surface_call( + Action::Type, + session, + "type", + Some(json!({ "nodeKey": node_key, "text": text, "clear": clear })), + ) + .await + { + Ok(payload) => ok_response( + action, + json!({ "sessionId": session.session_id, "targetId": target_id, "result": payload }), + format!( + "Entered {} characters into {target_id}.", + text.chars().count() + ), + ), + Err(response) => response, + } +} + +async fn press_key(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Key.as_str(); + let Some(key) = read_string(input, "key") else { + return missing_param(action, "key"); + }; + let modifiers = input + .get("modifiers") + .and_then(Value::as_array) + .map(|modifiers| { + modifiers + .iter() + .filter_map(Value::as_str) + .map(|value| value.to_ascii_lowercase()) + .filter(|value| matches!(value.as_str(), "alt" | "control" | "meta" | "shift")) + .collect::>() + }) + .unwrap_or_default(); + // A game reads a held key on its own animation frame, so a zero-length press + // is invisible to it. `holdMs` keeps the key down long enough to matter. + let hold_ms = input + .get("holdMs") + .and_then(Value::as_f64) + .filter(|value| value.is_finite() && *value > 0.0) + .map(|value| value as u64) + .unwrap_or(0) + .min(MINIAPP_WEB_MAX_KEY_HOLD_MS); + match surface_call( + Action::Key, + session, + "key", + Some(json!({ "key": key, "modifiers": modifiers, "holdMs": hold_ms })), + ) + .await + { + Ok(payload) => ok_response( + action, + json!({ "sessionId": session.session_id, "key": key, "holdMs": hold_ms, "result": payload }), + match hold_ms { + 0 => format!("Sent key {key}."), + _ => format!("Held key {key} for {hold_ms}ms."), + }, + ), + Err(response) => response, + } +} + +async fn scroll(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Scroll.as_str(); + let delta_x = input.get("deltaX").and_then(Value::as_f64).unwrap_or(0.0); + let delta_y = input.get("deltaY").and_then(Value::as_f64).unwrap_or(0.0); + if delta_x == 0.0 && delta_y == 0.0 { + return err_response( + action, + MiniAppWebErrorCode::InvalidParams, + "scroll requires a non-zero deltaX or deltaY.", + &[], + ); + } + let node_key = match read_string(input, "targetId") { + Some(target_id) => match resolve_target(&session.session_id, &target_id).await { + Ok((node_key, _summary)) => Some(node_key), + Err(code) => return target_error(action, code), + }, + None => None, + }; + match surface_call( + Action::Scroll, + session, + "scroll", + Some(json!({ "nodeKey": node_key, "deltaX": delta_x, "deltaY": delta_y })), + ) + .await + { + Ok(payload) => ok_response( + action, + json!({ + "sessionId": session.session_id, + "scroll": { "deltaX": delta_x, "deltaY": delta_y }, + "result": payload, + }), + format!("Scrolled deltaX={delta_x} deltaY={delta_y}."), + ), + Err(response) => response, + } +} + +/// `wait` never touches the surface: animations, network calls, and game loops +/// only need an observation interval, and the model re-observes afterwards. +async fn wait(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let requested = input + .get("durationMs") + .and_then(Value::as_f64) + .filter(|value| value.is_finite() && *value >= 0.0) + .map(|value| value as u64) + .unwrap_or(MINIAPP_WEB_DEFAULT_WAIT_MS) + .min(MINIAPP_WEB_MAX_WAIT_MS); + tokio::time::sleep(Duration::from_millis(requested)).await; + ok_response( + Action::Wait.as_str(), + json!({ "sessionId": session.session_id, "waitedMs": requested }), + format!("Waited {requested}ms. Re-run inspect to observe the updated page."), + ) +} + +/// Capture the MiniApp frame and hand the pixels to the model. +/// +/// A canvas or game MiniApp has no readable structure, so this is the only live +/// observation available: the Agent looks at what the app is drawing right now, +/// then acts, then looks again. Sampling is turn-paced, which is the honest +/// ceiling for a tool-using model; anything faster has to be a MiniApp-declared +/// bridge action that runs inside the app. +async fn screenshot(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::Screenshot.as_str(); + let max_edge = input + .get("maxEdge") + .and_then(Value::as_f64) + .filter(|value| value.is_finite() && *value >= 0.0) + .map(|value| value as u32) + .unwrap_or(MINIAPP_WEB_DEFAULT_CAPTURE_EDGE) + .clamp(320, MINIAPP_WEB_MAX_CAPTURE_EDGE); + let payload = match surface_call( + Action::Screenshot, + session, + "capture", + Some(json!({ "maxEdge": max_edge })), + ) + .await + { + Ok(payload) => payload, + Err(response) => return response, + }; + + let capture = match MiniAppWebCapture::parse(&payload) { + Ok(capture) => capture, + Err(code) => { + return err_response( + action, + code, + "The surface returned an unusable frame capture.", + &["Re-run screenshot; if it repeats, this MiniApp paints nothing that can be captured."], + ) + } + }; + + let summary = format!( + "Captured {}x{} from {} ({}). This is one frame at capture time; wait, act, then screenshot again to follow the app state.", + capture.width, + capture.height, + capture.label.as_deref().unwrap_or(&capture.source), + capture.mime_type + ); + ok_response_with_image( + action, + json!({ + "sessionId": session.session_id, + "width": capture.width, + "height": capture.height, + "source": capture.source, + "label": capture.label, + "mimeType": capture.mime_type, + "bytes": capture.decoded_len(), + }), + summary, + crate::util::types::ToolImageAttachment { + mime_type: capture.mime_type, + data_base64: capture.data_base64, + }, + ) +} + +async fn observe_bridge(session: &MiniAppWebSessionSnapshot) -> Vec { + let action = Action::ObserveBridge.as_str(); + // Automatic observation needs no app declaration, so only a frame that + // announced an older bridge without either capability is refused. + let can_observe = + session.has_bridge_capability("observe") || session.has_bridge_capability("auto-observe"); + if session.bridge_announced() && !can_observe { + return err_response( + action, + MiniAppWebErrorCode::BridgeUnavailable, + "This MiniApp announced no observation capability.", + &[ + "Only the MiniApp decides which internal state is Agent-visible; the host will not guess variable names.", + "DOM actions remain available if the surface reports dom.", + ], + ); + } + match surface_call( + Action::ObserveBridge, + session, + "bridge_observe", + Some(json!({ "requestId": request_id("observe") })), + ) + .await + { + Ok(payload) => { + store_bridge_capabilities(session, &payload).await; + let state = payload.get("state").cloned().unwrap_or(Value::Null); + let auto = payload.get("auto").cloned().unwrap_or(Value::Null); + let actions = payload + .get("actions") + .cloned() + .unwrap_or_else(|| json!(session.declared_actions)); + let summary = match state.is_null() { + true => format!( + "No declared state ({}). Automatic observation: {}", + session.app_id, + describe_auto_observation(&auto) + ), + false => format!( + "Bridge state: {}\nAutomatic observation: {}", + state, + describe_auto_observation(&auto) + ), + }; + let summary = format!("{summary}\n{}", describe_declared_actions(&actions)); + ok_response( + action, + json!({ + "sessionId": session.session_id, + "state": state, + "auto": auto, + "bridgeCapabilities": payload + .get("capabilities") + .cloned() + .unwrap_or_else(|| json!(session.bridge_capabilities)), + "declaredActions": actions, + }), + summary, + ) + } + Err(response) => response, + } +} + +/// Render the declared action vocabulary in the same one-line shape as the rest +/// of an observation, so the Agent never has to guess action names. +fn describe_declared_actions(actions: &Value) -> String { + let names: Vec<&str> = actions + .as_array() + .map(|items| items.iter().filter_map(Value::as_str).collect()) + .unwrap_or_default(); + if names.is_empty() { + return "declared actions: none (the app accepts action types it handles; send one and read the answer)." + .to_string(); + } + let mut joined = names.join(", "); + if joined.chars().count() > 240 { + joined = joined.chars().take(240).collect::() + " ..."; + } + format!("declared actions: {joined}") +} + +/// Render the automatic half of an observation in one glanceable line: what the +/// app draws as text, whether its loop is alive, and whether input arrived. +fn describe_auto_observation(auto: &Value) -> String { + if auto.is_null() { + return "none (the MiniApp did not report automatic observation).".to_string(); + } + let mut parts = Vec::new(); + let texts: Vec = auto + .get("canvasText") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .filter_map(|item| item.get("text").and_then(Value::as_str)) + .map(|text| text.trim().to_string()) + .filter(|text| !text.is_empty()) + .collect() + }) + .unwrap_or_default(); + if texts.is_empty() { + parts.push("drawn text: none yet".to_string()); + } else { + let mut joined = texts.join(" | "); + if joined.chars().count() > 240 { + joined = joined.chars().take(240).collect::() + " ..."; + } + parts.push(format!("drawn text: {joined}")); + } + match auto.get("frameAgeMs").and_then(Value::as_u64) { + Some(age) => parts.push(format!("animation frame {age}ms ago")), + None => parts.push("no animation frames yet (static or paused)".to_string()), + } + if let Some(input) = auto.get("input").filter(|value| !value.is_null()) { + let kind = input.get("type").and_then(Value::as_str).unwrap_or("input"); + let key = input + .get("key") + .map(Value::to_string) + .unwrap_or_else(|| "null".to_string()); + parts.push(format!("last input: {kind} {key}")); + } + parts.join(", ") +} + +/// Sample the bridge over a bounded window and report the timeline. +/// +/// A running app changes faster than a turn can read it, and calling +/// `observe_bridge` in a loop costs one model round per sample. This owns that +/// loop so one action replaces a whole turn-paced conversation, and it samples +/// through `observe` only: whatever presentation is hosting the app — a tab or +/// its own window — stays a transport detail below this line. +async fn watch_state(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::WatchState.as_str(); + let can_observe = + session.has_bridge_capability("observe") || session.has_bridge_capability("auto-observe"); + if session.bridge_announced() && !can_observe { + return err_response( + action, + MiniAppWebErrorCode::BridgeUnavailable, + "This MiniApp announced no observation capability.", + &["Without observable state there is nothing to sample; use screenshot for pixels."], + ); + } + let duration_ms = read_u64(input, "durationMs") + .unwrap_or(MINIAPP_WEB_DEFAULT_WATCH_MS) + .clamp(MINIAPP_WEB_MIN_WATCH_POLL_MS, MINIAPP_WEB_MAX_WATCH_MS); + let poll_ms = read_u64(input, "pollMs") + .unwrap_or(MINIAPP_WEB_DEFAULT_WATCH_POLL_MS) + .clamp(MINIAPP_WEB_MIN_WATCH_POLL_MS, duration_ms); + + let started = std::time::Instant::now(); + let mut samples: Vec = Vec::new(); + let mut last_key = String::new(); + let mut budget = 0usize; + let mut declared = false; + let mut stop_reason = "duration"; + let mut warning: Option = None; + // The session snapshot predates this call; a sample can announce capabilities + // the caller did not know yet, and the timeline must report what it observed. + let mut capabilities = session.bridge_capabilities.clone(); + + loop { + let payload = match surface_call( + Action::WatchState, + session, + "bridge_observe", + Some(json!({ "requestId": request_id("watch") })), + ) + .await + { + Ok(payload) => payload, + // A failure before the first sample is the outcome. After that, the + // timeline already collected is evidence, so keep it and say why the + // window ended early. + Err(response) if samples.is_empty() => return response, + Err(response) => { + warning = response[0] + .content() + .get("error") + .and_then(|error| error.get("message")) + .and_then(Value::as_str) + .map(str::to_string); + stop_reason = "error"; + break; + } + }; + store_bridge_capabilities(session, &payload).await; + if let Some(observed) = + payload + .get("capabilities") + .and_then(Value::as_array) + .map(|values| { + values + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect::>() + }) + { + capabilities = observed; + } + let auto = payload.get("auto").cloned().unwrap_or(Value::Null); + declared = auto.get("source").and_then(Value::as_str) == Some("declared"); + // A declared state is authoritative; otherwise the sampled value is the + // compact form of automatic observation. + let value = match payload.get("state") { + Some(state) if !state.is_null() => state.clone(), + _ => compact_auto(&auto), + }; + let key = value.to_string(); + // The first sample always lands, so an app that never changes still + // proves it was observed; later samples only appear when state moved. + if samples.is_empty() || key != last_key { + last_key = key; + budget += value.to_string().len(); + samples.push(json!({ "t": started.elapsed().as_millis() as u64, "state": value })); + if samples.len() >= MINIAPP_WEB_MAX_WATCH_SAMPLES { + stop_reason = "cap"; + break; + } + if budget >= MINIAPP_WEB_MAX_WATCH_SAMPLE_BYTES { + stop_reason = "size"; + break; + } + } + let elapsed = started.elapsed().as_millis() as u64; + if elapsed >= duration_ms { + break; + } + let remaining = duration_ms - elapsed; + tokio::time::sleep(Duration::from_millis(poll_ms.min(remaining))).await; + } + + let count = samples.len(); + let mut body = json!({ + "sessionId": session.session_id, + "durationMs": duration_ms, + "pollMs": poll_ms, + "sampleCount": count, + "stopReason": stop_reason, + "declared": declared, + "samples": samples, + "bridgeCapabilities": capabilities, + }); + if let Some(warning) = &warning { + body["warning"] = json!(warning); + } + ok_response( + action, + body, + format!( + "Sampled {} change(s) over {duration_ms}ms at {poll_ms}ms intervals ({stop_reason}). {}", + count, + match declared { + true => "Samples are the app's declared state.", + false => "Samples are automatic observation; the app declared no state.", + } + ), + ) +} + +/// Keep one sampled observation small enough that a timeline of them still fits +/// a single tool response: what the app draws, whether its loop is alive, and +/// the last input are the values that move while it runs. +fn compact_auto(auto: &Value) -> Value { + let texts: Vec = auto + .get("canvasText") + .and_then(Value::as_array) + .map(|items| { + items + .iter() + .take(16) + .filter_map(|item| item.get("text").and_then(Value::as_str)) + .map(|text| json!(text)) + .collect() + }) + .unwrap_or_default(); + json!({ + "source": auto.get("source").cloned().unwrap_or_else(|| json!("auto")), + "canvasText": texts, + "frames": auto.get("frames").cloned().unwrap_or(Value::Null), + "frameAgeMs": auto.get("frameAgeMs").cloned().unwrap_or(Value::Null), + "input": auto.get("input").cloned().unwrap_or(Value::Null), + }) +} + +async fn act_bridge(session: &MiniAppWebSessionSnapshot, input: &Value) -> Vec { + let action = Action::ActBridge.as_str(); + if session.bridge_announced() && !session.has_bridge_capability("act") { + return err_response( + action, + MiniAppWebErrorCode::BridgeUnavailable, + "This MiniApp does not declare the 'act' agent bridge capability.", + &[], + ); + } + let Some(bridge_action) = input.get("bridgeAction").cloned() else { + return missing_param(action, "bridgeAction"); + }; + let has_type = bridge_action + .get("type") + .and_then(Value::as_str) + .map(str::trim) + .is_some_and(|value| !value.is_empty()); + if !has_type { + return err_response( + action, + MiniAppWebErrorCode::InvalidParams, + "act_bridge requires bridgeAction.type.", + &[], + ); + } + if validate_bridge_payload_size(&bridge_action).is_err() { + return err_response( + action, + MiniAppWebErrorCode::InvalidParams, + "The bridge action payload is too large.", + &[], + ); + } + match surface_call( + Action::ActBridge, + session, + "bridge_act", + Some(json!({ + "requestId": request_id("act"), + "action": bridge_action, + })), + ) + .await + { + Ok(payload) => { + store_bridge_capabilities(session, &payload).await; + let result = payload.get("result").cloned().unwrap_or(Value::Null); + let summary = match result.is_null() { + true => "Bridge action completed.".to_string(), + false => format!("Bridge action result: {result}"), + }; + ok_response( + action, + json!({ "sessionId": session.session_id, "result": result }), + summary, + ) + } + Err(response) => response, + } +} + +/// The MiniApp is authoritative for what it declared; keep the session in step +/// so later calls gate on the announcement instead of a stale `open` snapshot. +async fn store_bridge_capabilities(session: &MiniAppWebSessionSnapshot, payload: &Value) { + if let Some(capabilities) = payload.get("capabilities") { + if let Ok(capabilities) = serde_json::from_value::>(capabilities.clone()) { + refresh_bridge_capabilities(&session.session_id, capabilities).await; + } + } + if let Some(actions) = payload.get("actions") { + if let Ok(actions) = serde_json::from_value::>(actions.clone()) { + refresh_bridge_actions(&session.session_id, actions).await; + } + } +} + +fn request_id(method: &str) -> String { + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT: AtomicU64 = AtomicU64::new(1); + format!( + "miniapp-{method}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + ) +} + +fn read_string(input: &Value, key: &str) -> Option { + input + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) +} + +fn read_u64(input: &Value, key: &str) -> Option { + input.get(key).and_then(|value| match value { + Value::Number(number) => number.as_u64(), + Value::String(text) => text.trim().parse().ok(), + _ => None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn session(capabilities: &[&str], bridge: &[&str]) -> MiniAppWebSessionSnapshot { + MiniAppWebSessionSnapshot { + session_id: "miniapp-session-1".to_string(), + app_id: "demo".to_string(), + instance_id: None, + strict_runtime: false, + width: None, + height: None, + capabilities: capabilities.iter().map(|value| value.to_string()).collect(), + bridge_capabilities: bridge.iter().map(|value| value.to_string()).collect(), + declared_actions: Vec::new(), + presented_from: None, + presentation: None, + target_count: 0, + } + } + + /// A host whose declared state moves on its own, standing in for a running app. + struct MovingHost { + calls: std::sync::atomic::AtomicUsize, + } + + #[async_trait::async_trait] + impl crate::agentic::tools::miniapp_surface::MiniAppWebHost for MovingHost { + async fn open( + &self, + request: crate::agentic::tools::miniapp_surface::MiniAppWebOpenRequest, + ) -> Result + { + use openbitfun_product_domains::miniapp::web_surface::MiniAppWebSurfaceInfo; + Ok(MiniAppWebSurfaceInfo { + app_id: request.app_id, + ..Default::default() + }) + } + + async fn execute(&self, _command: MiniAppWebCommand) -> Result { + let call = self + .calls + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + // Every other read repeats the previous value, so deduplication is + // observable instead of being masked by a fast-changing fake. + Ok(json!({ + "state": { "score": call / 2 }, + "auto": { "source": "declared" }, + "capabilities": ["observe", "act"], + })) + } + } + + #[tokio::test] + async fn watch_state_samples_a_timeline_and_drops_repeats() { + let _guard = crate::agentic::tools::miniapp_surface::HOST_SLOT_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + crate::agentic::tools::miniapp_surface::set_miniapp_web_host(std::sync::Arc::new( + MovingHost { + calls: std::sync::atomic::AtomicUsize::new(0), + }, + )); + let response = watch_state( + &session(&[], &[]), + &json!({ "durationMs": 800, "pollMs": 250 }), + ) + .await; + let body = response[0].content(); + assert_eq!(body["ok"], true, "{body}"); + let data = &body["data"]; + assert_eq!(data["stopReason"], "duration"); + assert_eq!(data["declared"], true, "a declared state is authoritative"); + assert_eq!(data["bridgeCapabilities"], json!(["observe", "act"])); + let samples = data["samples"].as_array().expect("samples"); + assert!(samples.len() >= 2, "the timeline must move: {body}"); + assert_eq!( + samples[0]["state"]["score"], 0, + "the first sample always lands" + ); + assert!( + samples.len() <= 4, + "repeats are dropped instead of padding the timeline: {body}" + ); + let last = samples.last().expect("last sample"); + assert!(last["state"]["score"].as_u64() > samples[0]["state"]["score"].as_u64()); + assert!(last["t"].as_u64().unwrap_or(0) > 0); + } + + #[test] + fn action_parsing_matches_the_published_list() { + for name in ACTIONS { + assert!(Action::parse(name).is_some(), "{name} must parse"); + } + assert_eq!(Action::parse("evaluate"), None); + assert_eq!( + action_from_input(&json!({ "action": " inspect " })), + Some(Action::Inspect) + ); + assert_eq!(action_from_input(&json!({})), None); + } + + #[test] + fn dom_gate_only_blocks_document_actions_on_opaque_surfaces() { + let opaque = session(&[], &["observe"]); + assert!(dom_gate(Action::Inspect, &opaque).is_some()); + assert!(dom_gate(Action::Click, &opaque).is_some()); + assert!(dom_gate(Action::Wait, &opaque).is_none()); + assert!(dom_gate(Action::ObserveBridge, &opaque).is_none()); + + let readable = session(&["dom", "input"], &[]); + assert!(dom_gate(Action::Inspect, &readable).is_none()); + } + + #[test] + fn a_window_presentation_reports_the_transport_gap_not_a_missing_bridge() { + let mut window = session(&[], &["auto-observe"]); + window.presentation = Some("window".to_string()); + let response = dom_gate(Action::Screenshot, &window).expect("window blocks pixels"); + let body = response[0].content(); + assert_eq!(body["error"]["code"], "CAPABILITY_MISSING"); + let message = body["error"]["message"].as_str().unwrap_or_default(); + assert!(message.contains("own window"), "{message}"); + assert!(body["error"]["hints"][0] + .as_str() + .unwrap_or_default() + .contains("act_bridge")); + // The bridge actions themselves are never gated by a missing DOM. + assert!(dom_gate(Action::ObserveBridge, &window).is_none()); + assert!(dom_gate(Action::WatchState, &window).is_none()); + // Key input is relayed by the independent Shell, so Canvas games do not + // need to be moved into a tab just to receive ArrowRight/Space. + assert!(dom_gate(Action::Key, &window).is_none()); + } + + #[tokio::test] + async fn missing_targets_are_reported_before_reaching_the_surface() { + let response = node_key(Action::Click, &session(&["dom"], &[]), &json!({})).await; + assert!(response.is_err(), "targetId is required"); + let body = response.err().expect("error response")[0].content(); + assert_eq!(body["error"]["code"], "INVALID_PARAMS"); + } + + #[test] + fn blank_strings_never_count_as_parameters() { + assert_eq!(read_string(&json!({ "appId": " " }), "appId"), None); + assert_eq!( + read_string(&json!({ "appId": " demo " }), "appId"), + Some("demo".to_string()) + ); + } + + #[test] + fn auto_observation_summarizes_drawn_text_and_liveness() { + let auto = json!({ + "canvasText": [{ "text": "SCORE 1200", "x": 10, "y": 20 }, { "text": "WAVE 3", "x": 10, "y": 40 }], + "frameAgeMs": 16, + "input": { "type": "keydown", "key": "ArrowLeft", "at": 1 }, + }); + let summary = describe_auto_observation(&auto); + assert!(summary.contains("SCORE 1200 | WAVE 3"), "{summary}"); + assert!(summary.contains("animation frame 16ms ago"), "{summary}"); + assert!(summary.contains("keydown"), "{summary}"); + + let idle = json!({ "canvasText": [], "frameAgeMs": null, "input": null }); + let summary = describe_auto_observation(&idle); + assert!(summary.contains("drawn text: none yet"), "{summary}"); + assert!(summary.contains("no animation frames yet"), "{summary}"); + assert!(describe_auto_observation(&Value::Null).contains("none")); + } + + #[test] + fn automatic_observation_needs_no_declared_capability() { + // The injected runtime always announces auto-observe, so a session that + // never declared state must not be treated as unobservable. + let auto_only = session(&["dom"], &["auto-observe"]); + assert!(auto_only.has_bridge_capability("auto-observe")); + assert!(!auto_only.has_bridge_capability("observe")); + assert!(!auto_only.has_bridge_capability("act")); + assert!(dom_gate(Action::ObserveBridge, &auto_only).is_none()); + assert!(dom_gate(Action::ActBridge, &auto_only).is_none()); + } + + #[tokio::test] + async fn watch_state_needs_observation_and_never_needs_a_dom() { + // A bridge that only exposes actions has nothing to sample, and the refusal + // must happen before any surface call. + let actions_only = session(&[], &["act"]); + let refused = watch_state(&actions_only, &json!({ "durationMs": 1000 })).await; + let body = refused[0].content(); + assert_eq!(body["error"]["code"], "BRIDGE_UNAVAILABLE"); + + // Sampling is state-based, so an opaque frame with automatic observation is + // allowed through the gate. + assert!(!Action::WatchState.requires_dom()); + assert!(dom_gate(Action::WatchState, &session(&[], &["auto-observe"])).is_none()); + } + + #[test] + fn the_declared_action_vocabulary_reaches_the_model() { + // The app's own words, in announcement order, are what the Agent needs to + // stop guessing action names. + let declared = json!(["start", "restart", "autoplay"]); + let summary = describe_declared_actions(&declared); + assert!(summary.contains("start, restart, autoplay"), "{summary}"); + + // An app that declares nothing is reported as unknown, never as "accepts + // nothing": undeclared names must stay attemptable. + let empty = describe_declared_actions(&json!([])); + assert!(empty.contains("declared actions: none"), "{empty}"); + assert!(empty.contains("read the answer"), "{empty}"); + assert!(describe_declared_actions(&Value::Null).contains("none")); + + // A verbose app cannot flood the turn with one observation. + let many = json!((0..40) + .map(|index| format!("action-{index}")) + .collect::>()); + assert!(describe_declared_actions(&many).chars().count() <= 300); + } + + #[test] + fn app_hints_stay_bounded_and_actionable() { + let apps = (0..14) + .map(|index| MiniAppWebAppSummary { + app_id: format!("app-{index}"), + name: format!("App {index}"), + ..Default::default() + }) + .collect::>(); + let hints = available_app_hints(&apps); + assert_eq!(hints.len(), 1); + assert!(hints[0].contains("App 0 (app-0)")); + assert!(hints[0].contains("4 more"), "the tail must be summarized"); + assert_eq!( + available_app_hints(&[]), + vec!["No MiniApps are installed in this profile.".to_string()] + ); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/mod.rs b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/mod.rs new file mode 100644 index 0000000000..d0520c52e1 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/mod.rs @@ -0,0 +1,342 @@ +//! MiniAppBrowser tool — drive a MiniApp the way BrowserUse drives a Web page. +//! +//! A MiniApp is a Web page hosted by the product window, so the Agent controls +//! it with inspect / locate / input / wait semantics instead of editing source +//! or touching host paths. Layering: +//! +//! - [`crate::agentic::tools::miniapp_surface`] owns the host +//! port, sessions, target handles, capability facts, and the method whitelist. +//! - [`actions`] owns what each Agent action means and forwards it. +//! - [`result`] owns the model-facing envelopes and node rendering. +//! - this module owns only the `Tool` contract surface. + +mod actions; +mod result; + +use crate::agentic::tools::framework::{ + Tool, ToolExposure, ToolRenderOptions, ToolResult, ToolUseContext, ValidationResult, +}; +use crate::agentic::tools::miniapp_surface::miniapp_web_host_available; +use crate::util::errors::OpenBitFunResult; +use actions::{action_from_input, dispatch, Action, ACTIONS}; +use async_trait::async_trait; +use openbitfun_product_domains::miniapp::web_surface::{ + MINIAPP_WEB_DEFAULT_CAPTURE_EDGE, MINIAPP_WEB_DEFAULT_WAIT_MS, MINIAPP_WEB_MAX_CAPTURE_EDGE, + MINIAPP_WEB_MAX_DEPTH, MINIAPP_WEB_MAX_NODES, MINIAPP_WEB_MAX_WAIT_MS, + MINIAPP_WEB_MAX_WATCH_MS, +}; +use serde_json::{json, Value}; + +const TOOL_NAME: &str = "MiniAppBrowser"; + +pub struct MiniAppBrowserTool; + +impl MiniAppBrowserTool { + pub fn new() -> Self { + Self + } +} + +impl Default for MiniAppBrowserTool { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Tool for MiniAppBrowserTool { + fn name(&self) -> &str { + TOOL_NAME + } + + async fn description(&self) -> OpenBitFunResult { + Ok(format!( + r#"Operate an installed MiniApp like a Web page: open it, read its structure, locate elements, then click, type, send keys, or scroll. This is the primary way to use a MiniApp; do not modify its source unless the user asks. + +`open` drives the same copy the user sees, and picks where that copy lives in this order: the app's own window if it is already open, the tab already showing it, the desktop app the user published from this MiniApp (launched as a normal app), and only then a new tab. Never launch a copy of the app in a separate or headless browser: that runs something the user cannot see, and its state never matches the app on their screen. + +Addressing an app: call `apps` when you do not know the id. It lists installed MiniApps with the id that addresses each one. `open` accepts `appId` or the app's `name`, resolves an exact id or name first, accepts a unique partial name, and reports candidates instead of guessing when several apps match. If the app is already open in one of its own windows or in a tab, `open` attaches to that presentation instead of starting a second copy, so you drive the instance the user has on screen. + +Typical loop: open(appId) -> inspect -> find (or reuse targetIds from inspect) -> click/type/read -> wait -> inspect. +For any MiniApp task, first check whether the user has provided or authorized access to the app source through the available workspace/context. When source is available, inspect the relevant UI, data, state transitions, and action handlers; derive the shortest reliable sequence that fulfills the request and execute it directly. Do not spend turns repeatedly probing behavior that source inspection can settle. Use the live MiniApp to enter the workflow and confirm completion or meaningful checkpoints. If source is unavailable, use DOM or declared bridge actions to identify a direct path; use observe-act cycles only for genuinely dynamic or uncertain steps. Prefer structured state over pixels; screenshot is a fallback only when the available state cannot confirm progress. + +Following something that keeps changing: `watch_state` samples the app's own state inside one call (window up to {MINIAPP_WEB_MAX_WATCH_MS}ms, deduplicated by state) and returns the timeline. Use it instead of calling `observe_bridge` in a loop, and instead of writing a harness script. Do not load the app's HTML in a separate browser, Chrome, CDP, or headless run to "verify" it: that renders a copy the user cannot see, on a different clock, with no bridge and no host state, so its result is not evidence about the app they have open. If you need a long observation window, ask `watch_state` for it. + +Semantics: +- One sessionId is bound to one appId. Later calls only act on that app; reopening the app replaces the older session. +- inspect returns a bounded role/name/testId tree with a short-lived targetId per node. Defaults: at most {MINIAPP_WEB_MAX_NODES} nodes and depth {MINIAPP_WEB_MAX_DEPTH}. +- find locator priority: testId, then accessibility role+name, then name/text, then a CSS selector as a compatibility fallback. +- targetIds expire (10 minutes), die on reload or re-render, and never cross MiniApps. On STALE_TARGET or UNKNOWN_TARGET, call inspect again. +- wait gives animations, network calls, and game loops an observation interval ({MINIAPP_WEB_DEFAULT_WAIT_MS}ms default, {MINIAPP_WEB_MAX_WAIT_MS}ms max); re-observe afterwards. +- screenshot captures the largest canvas in the frame now (longest side {MINIAPP_WEB_DEFAULT_CAPTURE_EDGE}px by default, {MINIAPP_WEB_MAX_CAPTURE_EDGE}px max) and returns it as an image you can look at. Use it only when DOM and bridge observations cannot answer the next decision; screenshots are slower, turn-paced samples rather than a live game feed. +- close releases only this control session. The app stays open for the user; dismiss it from the UI when the user asks. + +Capability honesty: +- A readable MiniApp reports dom/accessibility/input and supports the full loop with no source change. +- A canvas-backed MiniApp has no readable structure: `inspect` sees only the canvas element. Start with `observe_bridge` for drawn text and loop liveness, and use `watch_state` when a bounded timeline helps. Use `act_bridge` for app-declared semantic actions or `key`/`click` for ordinary controls. If those observations do not show enough state to choose an action, use `screenshot` as a fallback; sampling pixels is turn-paced, not continuous. +- A MiniApp running in the strict (opaque-origin) profile reports no DOM and its pixels cannot be read. It is still observable through `observe_bridge` (automatic observation, plus `state` when the app declares it) and actionable through `act_bridge` when the app declares actions. If the app declares nothing, automatic observation still reports drawn text, loop liveness, and input echo; say what is missing instead of guessing. +- A MiniApp presented in its own window (`surface.kind` is `window`) answers the bridge and host-routed key input, but not the document: `observe_bridge`, `watch_state`, `act_bridge`, and `key` work there, while `inspect`, `find`, `click`, `type`, `scroll`, and `screenshot` are refused with CAPABILITY_MISSING because a separate window cannot take synthetic DOM input or hand out pixels. That is a transport limit, not a broken app: use the app's bridge state plus `key`, or ask the user to move it to a tab when DOM/pixels are required. + +Scope: this tool cannot read host paths, credentials, or other MiniApps, and it has no `evaluate` action, so it cannot run arbitrary JavaScript. `screenshot` returns pixels only: it never exposes the app's variables, scores, or timers unless the app declares them through its bridge. + +Live state and auto-play: +- Observation works on an unchanged MiniApp. Every frame already reports automatic observations through `observe_bridge`: the text the app draws (canvas `fillText` positions and strings, which usually includes score, wave, and lives), whether its animation loop is still running (`frames`, `frameAgeMs`), whether dispatched input actually reached it (`input`, `inputCount`), plus viewport and canvas sizes for mapping coordinates onto a screenshot. `state` is null in that mode. No app edit, no host change, no manifest permission. +- An app that wants labeled, authoritative state announces it from inside itself: `window.app.agent.expose({{ observe: () => ({{ score, lives, wave }}), act: (action) => ... }})`. The `window.app` runtime is injected into every MiniApp frame, including the strict runtime. Once the app calls `expose`, `observe_bridge` returns that JSON under `state` and `act_bridge` runs the actions the app itself accepts. The same call may list `actions: ['restart', 'autoplay']`, the action types it answers, which then arrive as `declaredActions`. That is what makes a fast-moving app playable at full speed: read real state instead of sampling frames, and let the app resolve timing inside one call. +- Prefer automatic observation first. Add `expose` through the MiniApp authoring owner (update its source, then finalize the file edit) when drawn text is not enough to play reliably: expose bounded state such as player position, score, lives, and game phase, plus semantic actions such as jump/restart when the game can apply them itself. This is faster and more reliable than pixel sampling. Do not try to inject scripts from this tool; there is no `evaluate`. +- The action vocabulary belongs to the app. `observe_bridge` returns `declaredActions`, the action types the app said its `act` accepts, so read them there first and use those names; an empty list means the app declared nothing, not that it accepts nothing. Do not assume installed-app source is present in the workspace or bypass host boundaries to find it. When source has been explicitly provided or authorized and is available through the normal file/context tools, use it to plan instead of guessing. Keep source-derived actions bounded and execute them against the user's live instance; use bridge observations to confirm the outcome, not as a mandatory round trip after every input. +- A fresh `open` on the app's own window may report `bridgeAnnounced: false` and empty capabilities: the window answers the bridge, the document just has not declared anything yet in this session. `observe_bridge` is the authoritative answer, so call it before concluding the surface is inert. +- Without useful bridge state the limit is turn-paced: one small input, then observe again; use a screenshot only when pixels are needed. Say so plainly instead of claiming you can follow continuous motion."# + )) + } + + fn short_description(&self) -> String { + "Open, read, and interact with a MiniApp in the user's own window, as a Web page." + .to_string() + } + + fn default_exposure(&self) -> ToolExposure { + ToolExposure::Deferred + } + + fn input_schema(&self) -> Value { + json!({ + "type": "object", + "additionalProperties": false, + "required": ["action"], + "properties": { + "action": { "type": "string", "enum": ACTIONS }, + "appId": { "type": "string", "description": "open: installed MiniApp id. Call apps when the id is unknown." }, + "name": { "type": "string", "description": "open: installed MiniApp display name, used when the id is unknown. An exact name wins; a partial name must match exactly one app." }, + "sessionId": { "type": "string", "description": "Session returned by open. Required by every action except open, sessions, and apps." }, + "targetId": { "type": "string", "description": "Short-lived target returned by inspect or find." }, + "locator": { + "type": "object", + "description": "find: highest-priority axis first. Provide testId, or role with name, or text, or a selector fallback.", + "properties": { + "testId": { "type": "string" }, + "role": { "type": "string" }, + "name": { "type": "string" }, + "text": { "type": "string" }, + "selector": { "type": "string", "description": "Compatibility fallback only." }, + "exact": { "type": "boolean", "description": "Require exact name/text matches instead of substrings." } + }, + "additionalProperties": false + }, + "attribute": { "type": "string", "description": "read: optional attribute name to return alongside text and value." }, + "text": { "type": "string", "description": "type: text to enter into the target." }, + "clear": { "type": "boolean", "description": "type: replace the existing value (default true)." }, + "key": { "type": "string", "description": "key: DOM key name such as Enter, Escape, ArrowLeft, Space, or a single character." }, + "modifiers": { + "type": "array", + "items": { "type": "string", "enum": ["Alt", "Control", "Meta", "Shift"] }, + "description": "key: modifiers held while the key is sent." + }, + "holdMs": { "type": "number", "description": "key: keep the key down this long (0-2000ms) before releasing it. Game loops read held input, so a movement key usually needs a hold." }, + "maxEdge": { "type": "number", "description": "screenshot: longest side of the returned image in pixels (320-1024, default 900)." }, + "deltaX": { "type": "number", "description": "scroll: horizontal delta in pixels." }, + "deltaY": { "type": "number", "description": "scroll: vertical delta in pixels." }, + "durationMs": { "type": "number", "description": "wait: observation interval in milliseconds. watch_state: length of the sampling window (250-120000ms, default 20000)." }, + "pollMs": { "type": "number", "description": "watch_state: sampling interval inside the window (250ms-120000ms, default 1000). Samples are deduplicated by state, so a quiet app returns few." }, + "bridgeAction": { + "type": "object", + "description": "act_bridge: MiniApp-declared semantic action, for example {\"type\":\"restart\"}.", + "properties": { "type": { "type": "string" } }, + "required": ["type"], + "additionalProperties": true + } + } + }) + } + + fn is_readonly(&self) -> bool { + false + } + + fn is_concurrency_safe(&self, _input: Option<&Value>) -> bool { + false + } + + async fn is_enabled(&self) -> bool { + miniapp_web_host_available() + } + + async fn description_with_context( + &self, + context: Option<&ToolUseContext>, + ) -> OpenBitFunResult { + let mut description = self.description().await?; + if !miniapp_web_host_available() { + description.push_str( + "\n\nNo MiniApp Web surface host is installed in this runtime. Controlling a MiniApp requires an active Desktop product surface; do not attempt it from this session.", + ); + } else if context.map(ToolUseContext::is_remote).unwrap_or(false) { + description.push_str( + "\n\n**Remote workspace:** the MiniApp surface belongs to the controller window. This tool drives that controller-local surface and never proxies a remote app.", + ); + } + Ok(description) + } + + async fn validate_input( + &self, + input: &Value, + _context: Option<&ToolUseContext>, + ) -> ValidationResult { + if action_from_input(input).is_some() { + return ValidationResult::default(); + } + ValidationResult { + result: false, + message: Some(format!( + "Missing or unsupported action. Supported actions: {}.", + ACTIONS.join(", ") + )), + error_code: None, + meta: None, + } + } + + fn render_tool_use_message(&self, input: &Value, _options: &ToolRenderOptions) -> String { + match action_from_input(input) { + Some(action) => format!("{TOOL_NAME}: {}", action.as_str()), + None => TOOL_NAME.to_string(), + } + } + + fn render_result_for_assistant(&self, output: &Value) -> String { + if output.get("ok").and_then(Value::as_bool).unwrap_or(false) { + return output + .get("data") + .map(Value::to_string) + .unwrap_or_else(|| output.to_string()); + } + let error = &output["error"]; + let mut summary = format!( + "{}: {}", + error["code"].as_str().unwrap_or("HOST_ERROR"), + error["message"].as_str().unwrap_or_default() + ); + if let Some(hints) = error["hints"].as_array() { + for hint in hints.iter().filter_map(Value::as_str) { + summary.push_str("\n- "); + summary.push_str(hint); + } + } + summary + } + + async fn call_impl( + &self, + input: &Value, + context: &ToolUseContext, + ) -> OpenBitFunResult> { + let Some(action) = action_from_input(input) else { + return Ok(result::err_response( + "?", + openbitfun_product_domains::miniapp::web_surface::MiniAppWebErrorCode::InvalidParams, + "Missing or unsupported action.", + &[ACTIONS.join(", ").as_str()], + )); + }; + // Same gate the browser-use loop applies to page screenshots: a capture + // the model cannot see is not an observation, so refuse it up front + // instead of spending a round trip on pixels that get dropped. + if action == Action::Screenshot { + if let Some(reason) = context.image_observation_unavailable_reason() { + return Ok(result::ok_response( + action.as_str(), + json!({ + "sessionId": input.get("sessionId").cloned().unwrap_or(Value::Null), + "screenshotUnavailable": true, + "reason": reason, + }), + format!( + "Screenshots cannot reach this session's model ({reason}). Observe a readable MiniApp with inspect and read, or ask the app for bridge state via observe_bridge. Do not retry screenshot." + ), + )); + } + } + Ok(dispatch(action, input).await) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::agentic::tools::implementations::miniapp_browser::actions::Action; + + #[test] + fn schema_and_dispatcher_agree_on_the_action_set() { + let schema = MiniAppBrowserTool::new().input_schema(); + let published = schema["properties"]["action"]["enum"] + .as_array() + .expect("action enum") + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect::>(); + assert_eq!( + published, + ACTIONS + .iter() + .map(|name| name.to_string()) + .collect::>() + ); + } + + #[test] + fn document_actions_are_behind_the_readable_surface_gate_but_key_is_not() { + for action in [ + Action::Inspect, + Action::Find, + Action::Read, + Action::Click, + Action::Type, + Action::Scroll, + // Pixels can only be read from a frame this window may inspect, so a + // strict-runtime MiniApp cannot be screenshotted either. + Action::Screenshot, + ] { + assert!( + action.requires_dom(), + "{action:?} must require a readable DOM" + ); + } + for action in [ + Action::Key, + Action::Wait, + Action::ObserveBridge, + Action::ActBridge, + ] { + assert!(!action.requires_dom(), "{action:?} must not require a DOM"); + } + } + + #[tokio::test] + async fn unknown_actions_are_rejected_during_validation() { + let tool = MiniAppBrowserTool::new(); + let validation = tool.validate_input(&json!({}), None).await; + assert!(!validation.result); + assert!(validation + .message + .unwrap_or_default() + .contains("observe_bridge")); + + let accepted = tool + .validate_input(&json!({ "action": "inspect" }), None) + .await; + assert!(accepted.result); + } + + #[tokio::test] + async fn description_prefers_source_derived_workflows_when_authorized() { + let description = MiniAppBrowserTool::new() + .description() + .await + .expect("tool description"); + assert!(description.contains("For any MiniApp task")); + assert!(description.contains("derive the shortest reliable sequence")); + assert!(description.contains("genuinely dynamic or uncertain steps")); + assert!(description.contains("Do not assume installed-app source is present")); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/result.rs b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/result.rs new file mode 100644 index 0000000000..a767c6def7 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/result.rs @@ -0,0 +1,236 @@ +//! Model-facing envelopes and node rendering for the MiniAppBrowser tool. +//! +//! Every response keeps the same shape the rest of the Agent tools use +//! (`ok` / `action` / `data`, or `ok: false` with a typed `error`), and every +//! failure carries a stable code from [`MiniAppWebErrorCode`] so the model can +//! branch on it instead of on English text. + +use crate::agentic::tools::framework::ToolResult; +use crate::agentic::tools::miniapp_surface::{classify_surface_error, mint_target}; +use openbitfun_product_domains::miniapp::web_surface::{MiniAppWebErrorCode, MiniAppWebNode}; +use serde_json::{json, Value}; + +/// Cap on how many node lines a single summary carries, so an `inspect` of a +/// dense page still reads as an outline instead of flooding the transcript. +const MAX_SUMMARY_LINES: usize = 40; + +pub(super) fn ok_response(action: &str, data: Value, summary: String) -> Vec { + let body = json!({ + "ok": true, + "action": action, + "data": data, + }); + vec![ToolResult::ok(body, Some(summary))] +} + +/// Success that also carries one image the model can look at. Used by `screenshot`, +/// where the pixels *are* the observation. +pub(super) fn ok_response_with_image( + action: &str, + data: Value, + summary: String, + image: crate::util::types::ToolImageAttachment, +) -> Vec { + let body = json!({ + "ok": true, + "action": action, + "data": data, + }); + vec![ToolResult::ok_with_images(body, Some(summary), vec![image])] +} + +pub(super) fn err_response( + action: &str, + code: MiniAppWebErrorCode, + message: impl Into, + hints: &[&str], +) -> Vec { + let message = message.into(); + let mut summary = format!("{}: {message}", code.as_str()); + for hint in hints { + summary.push_str("\n- "); + summary.push_str(hint); + } + let body = json!({ + "ok": false, + "action": action, + "error": { + "code": code.as_str(), + "message": message, + "hints": hints, + }, + }); + vec![ToolResult::ok(body, Some(summary))] +} + +pub(super) fn missing_param(action: &str, param: &str) -> Vec { + err_response( + action, + MiniAppWebErrorCode::InvalidParams, + format!("{action} requires '{param}'."), + &[], + ) +} + +/// A target handle the surface runtime no longer knows about: reloaded, +/// re-rendered, expired, or minted by another MiniApp. +pub(super) fn target_error(action: &str, code: MiniAppWebErrorCode) -> Vec { + err_response( + action, + code, + "The target handle is no longer valid for this MiniApp surface.", + &["Call inspect or find again: targets die on reload, re-render, expiry, or app change."], + ) +} + +/// Translate a surface-runtime failure into a typed, actionable response. +pub(super) fn surface_error(action: &str, message: String) -> Vec { + let code = classify_surface_error(&message); + let hints: &[&str] = match code { + MiniAppWebErrorCode::StaleTarget | MiniAppWebErrorCode::UnknownTarget => &[ + "Call inspect or find again: the MiniApp reloaded or re-rendered and earlier targetIds are stale.", + ], + MiniAppWebErrorCode::CapabilityMissing => &[ + "This MiniApp frame exposes no readable DOM. Ask the MiniApp author to declare the optional agent bridge, or open it in a readable surface.", + ], + MiniAppWebErrorCode::BridgeUnavailable => &[ + "The MiniApp does not declare this bridge capability, so the host cannot read its internal state. Do not guess internal state.", + ], + MiniAppWebErrorCode::BridgeTimeout | MiniAppWebErrorCode::Timeout => &[ + "The MiniApp did not answer in time. Re-observe before acting again.", + ], + _ => &["Re-run inspect to resynchronize with the current page state."], + }; + err_response(action, code, message, hints) +} + +/// A rendered observation: the model-facing tree plus the outline lines used as +/// the assistant summary. +pub(super) struct RenderedNodes { + pub tree: Value, + pub lines: Vec, +} + +/// Mint one target handle per node and strip the host-only node keys. +/// +/// Target handles are minted in pre-order (sequentially, so handle numbering +/// follows reading order), then the tree is rebuilt. Depth is bounded by the +/// contract limits, so the rebuild recursion is safe. +pub(super) async fn render_nodes(session_id: &str, nodes: Vec) -> RenderedNodes { + let mut flattened = Vec::new(); + flatten(nodes.clone(), &mut flattened); + + let mut target_ids = Vec::with_capacity(flattened.len()); + for node in &flattened { + let target_id = match node.node_key.as_deref() { + Some(node_key) => mint_target(session_id, node_key, node.summary_line("")) + .await + .ok(), + None => None, + }; + target_ids.push(target_id); + } + + let mut cursor = 0; + let mut lines = Vec::new(); + let tree = build(nodes, &target_ids, &mut cursor, &mut lines); + RenderedNodes { tree, lines } +} + +fn flatten(nodes: Vec, out: &mut Vec) { + for node in nodes { + out.push(node.clone()); + flatten(node.children, out); + } +} + +fn build( + nodes: Vec, + target_ids: &[Option], + cursor: &mut usize, + lines: &mut Vec, +) -> Value { + let mut rendered = Vec::with_capacity(nodes.len()); + for node in nodes { + let index = *cursor; + *cursor += 1; + let target_id = target_ids.get(index).and_then(|value| value.clone()); + if let Some(target_id) = target_id.as_deref() { + if lines.len() < MAX_SUMMARY_LINES { + lines.push(node.summary_line(target_id)); + } + } + let children = node.children; + let mut value = json!({ + "targetId": target_id, + "role": node.role, + "name": node.name, + "tag": node.tag, + "testId": node.test_id, + "value": node.value, + "text": node.text, + "enabled": node.enabled, + "checked": node.checked, + }); + if !children.is_empty() { + value["children"] = build(children, target_ids, cursor, lines); + } + rendered.push(value); + } + Value::Array(rendered) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_envelopes_carry_a_code_and_never_read_as_success() { + let response = err_response( + "click", + MiniAppWebErrorCode::StaleTarget, + "target is stale", + &["call inspect again"], + ); + let body = response[0].content(); + assert_eq!(body["ok"], false); + assert_eq!(body["action"], "click"); + assert_eq!(body["error"]["code"], "STALE_TARGET"); + + let summary = match &response[0] { + ToolResult::Result { + result_for_assistant, + .. + } => result_for_assistant.clone().unwrap_or_default(), + other => panic!("unexpected tool result: {other:?}"), + }; + assert!(summary.starts_with("STALE_TARGET: target is stale")); + assert!(summary.contains("- call inspect again")); + } + + #[test] + fn ok_envelopes_carry_action_and_summary() { + let response = ok_response("sessions", json!({}), "no sessions".to_string()); + let body = response[0].content(); + assert_eq!(body["ok"], true); + assert_eq!(body["action"], "sessions"); + } + + #[test] + fn surface_errors_become_typed_codes() { + let body = surface_error("click", "STALE_TARGET: frame reloaded".to_string())[0].content(); + assert_eq!(body["error"]["code"], "STALE_TARGET"); + let body = surface_error("click", "everything is on fire".to_string())[0].content(); + assert_eq!(body["error"]["code"], "HOST_ERROR"); + } + + #[test] + fn target_errors_are_reported_as_invalid_handles() { + let body = target_error("read", MiniAppWebErrorCode::UnknownTarget)[0].content(); + assert_eq!(body["error"]["code"], "UNKNOWN_TARGET"); + assert!(body["error"]["hints"][0] + .as_str() + .unwrap_or_default() + .contains("inspect")); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs index e98569eab8..b6a9a3b961 100644 --- a/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs +++ b/src/crates/assembly/core/src/agentic/tools/implementations/mod.rs @@ -49,6 +49,8 @@ pub mod ls_tool; #[cfg(feature = "tools-mcp")] pub mod mcp_tools; #[cfg(feature = "tools-miniapp")] +pub mod miniapp_browser; +#[cfg(feature = "tools-miniapp")] pub mod miniapp_finalize_tool; #[cfg(feature = "tools-miniapp")] pub mod miniapp_init_tool; @@ -126,6 +128,8 @@ pub use mcp_tools::{ GetMCPPromptTool, ListMCPPromptsTool, ListMCPResourcesTool, ReadMCPResourceTool, }; #[cfg(feature = "tools-miniapp")] +pub use miniapp_browser::MiniAppBrowserTool; +#[cfg(feature = "tools-miniapp")] pub use miniapp_finalize_tool::FinalizeMiniAppTool; #[cfg(feature = "tools-miniapp")] pub use miniapp_init_tool::InitMiniAppTool; diff --git a/src/crates/assembly/core/src/agentic/tools/miniapp_surface.rs b/src/crates/assembly/core/src/agentic/tools/miniapp_surface.rs new file mode 100644 index 0000000000..4d7bce4355 --- /dev/null +++ b/src/crates/assembly/core/src/agentic/tools/miniapp_surface.rs @@ -0,0 +1,758 @@ +//! Platform-neutral host port for driving a MiniApp Web surface. +//! +//! A MiniApp is a Web page hosted by the product window, so the Agent drives it +//! with BrowserUse semantics instead of source edits or Tauri-only commands. +//! This module owns the transport-neutral parts: +//! +//! - the host port the Desktop adapter registers, +//! - the session registry (one session is bound to exactly one `appId`), +//! - short-lived target handles and their staleness rules, +//! - typed request/response normalization for the Web UI surface runtime. +//! +//! DOM traversal and event dispatch stay in the Web UI surface runtime; the host +//! port only forwards a bounded, typed request and validates what comes back. + +use async_trait::async_trait; +use openbitfun_product_domains::miniapp::web_surface::{ + validate_bridge_payload_size, MiniAppWebAppSummary, MiniAppWebCapability, MiniAppWebErrorCode, + MiniAppWebNode, MiniAppWebSurfaceInfo, MiniAppWebTargetRegistry, MINIAPP_WEB_MAX_APPS, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::HashMap; +use std::sync::{Arc, OnceLock, RwLock}; +use tokio::sync::Mutex; + +/// Actions the Web UI surface runtime must implement. Anything else is refused +/// before it reaches the frontend, so the Agent tool contract stays closed. +pub const SURFACE_METHODS: &[&str] = &[ + "inspect", + "find", + "read", + "click", + "type", + "key", + "scroll", + // Pixel observation: a canvas or game surface has no readable structure, so + // the frame itself is the only live state the Agent can perceive. + "capture", + "bridge_observe", + "bridge_act", +]; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MiniAppWebOpenRequest { + pub app_id: String, +} + +/// One forwarded surface request. `app_id`/`instance_id` identify the bound +/// surface so the frontend never resolves an ambiguous frame on its own. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MiniAppWebCommand { + pub app_id: String, + #[serde(default)] + pub instance_id: Option, + pub method: String, + #[serde(default)] + pub params: Option, +} + +impl MiniAppWebCommand { + pub fn new( + app_id: &str, + instance_id: Option<&str>, + method: &str, + params: Option, + ) -> Self { + Self { + app_id: app_id.to_string(), + instance_id: instance_id.map(str::to_string), + method: method.to_string(), + params, + } + } +} + +/// Desktop (or any product surface) implementation of the MiniApp Web surface. +#[async_trait] +pub trait MiniAppWebHost: Send + Sync { + /// Present the MiniApp in a surface the Agent can control and return its + /// capabilities. Must not silently fall back to a surface the controller + /// cannot reach. + async fn open(&self, request: MiniAppWebOpenRequest) -> Result; + + async fn execute(&self, command: MiniAppWebCommand) -> Result; + + /// Enumerate the MiniApps this host can present, so the Agent can address an + /// app the user named instead of inventing an id or reading host paths. + /// + /// The default refuses: a host that does not know its installed apps must + /// say so, because an empty list would read as "nothing is installed". + async fn list_apps(&self) -> Result, String> { + Err(format!( + "{}: this host cannot enumerate installed MiniApps", + MiniAppWebErrorCode::SurfaceUnavailable.as_str() + )) + } +} + +type HostSlot = RwLock>>; + +static MINIAPP_WEB_HOST: OnceLock = OnceLock::new(); +static MINIAPP_WEB_SESSIONS: OnceLock> = OnceLock::new(); + +fn host_slot() -> &'static HostSlot { + MINIAPP_WEB_HOST.get_or_init(|| RwLock::new(None)) +} + +fn read_lock(lock: &RwLock) -> std::sync::RwLockReadGuard<'_, T> { + lock.read().unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +fn write_lock(lock: &RwLock) -> std::sync::RwLockWriteGuard<'_, T> { + lock.write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) +} + +pub fn set_miniapp_web_host(host: Arc) { + *write_lock(host_slot()) = Some(host); +} + +/// Tests that mutate the process-global host slot must serialize, including the +/// ones in sibling modules, so this lives at module level instead of inside the +/// test module that happened to need it first. +#[cfg(test)] +pub(crate) static HOST_SLOT_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +pub fn miniapp_web_host_available() -> bool { + read_lock(host_slot()).is_some() +} + +pub fn miniapp_web_host() -> Result, String> { + read_lock(host_slot()).clone().ok_or_else(|| { + "The MiniApp Web surface host is unavailable in this runtime. Controlling a MiniApp requires an active Desktop product surface.".to_string() + }) +} + +fn sessions() -> &'static Mutex { + MINIAPP_WEB_SESSIONS.get_or_init(|| Mutex::new(SessionRegistry::default())) +} + +/// A live control session. The session id is the only handle the Agent keeps +/// afterwards, so it must stay bound to the app that opened it. +#[derive(Debug)] +pub struct MiniAppWebSession { + pub session_id: String, + pub surface: MiniAppWebSurfaceInfo, + targets: MiniAppWebTargetRegistry, +} + +impl MiniAppWebSession { + pub fn app_id(&self) -> &str { + &self.surface.app_id + } + + pub fn snapshot(&self) -> MiniAppWebSessionSnapshot { + MiniAppWebSessionSnapshot { + session_id: self.session_id.clone(), + app_id: self.surface.app_id.clone(), + instance_id: self.surface.instance_id.clone(), + strict_runtime: self.surface.strict_runtime, + width: self.surface.width, + height: self.surface.height, + capabilities: self.surface.capabilities.clone(), + bridge_capabilities: self.surface.bridge_capabilities.clone(), + declared_actions: self.surface.declared_actions.clone(), + presented_from: self.surface.presented_from.clone(), + presentation: self.surface.presentation.clone(), + target_count: self.targets.len(), + } + } + + pub fn to_value(&self) -> Value { + self.snapshot().to_value() + } + + /// Capability gate used by the tool before it forwards an action. + pub fn require_capability( + &self, + capability: MiniAppWebCapability, + ) -> Result<(), MiniAppWebErrorCode> { + if self.surface.has_capability(capability) { + Ok(()) + } else { + Err(MiniAppWebErrorCode::CapabilityMissing) + } + } +} + +/// Detached view of a session. Tool responses must not hold the registry lock, +/// so every read path copies what it needs. +#[derive(Clone, Debug)] +pub struct MiniAppWebSessionSnapshot { + pub session_id: String, + pub app_id: String, + pub instance_id: Option, + pub strict_runtime: bool, + pub width: Option, + pub height: Option, + pub capabilities: Vec, + pub bridge_capabilities: Vec, + /// Action types the app listed for `act`. Empty means it listed none, so the + /// vocabulary is unknown rather than empty: the app still decides what it + /// accepts, and an undeclared name is never blocked on this basis. + pub declared_actions: Vec, + /// Where the app was presented before this request, when the host had to move + /// it (for example out of its own window into a tab). + pub presented_from: Option, + /// `Some("window")` when the Agent controls the MiniApp's own window, which + /// answers the agent bridge but cannot receive DOM input or pixels. + pub presentation: Option, + pub target_count: usize, +} + +impl MiniAppWebSessionSnapshot { + pub fn has_capability(&self, capability: MiniAppWebCapability) -> bool { + self.capabilities + .iter() + .any(|value| value == capability.as_str()) + } + + pub fn has_dom(&self) -> bool { + self.has_capability(MiniAppWebCapability::Dom) + } + + pub fn has_bridge_capability(&self, capability: &str) -> bool { + self.bridge_capabilities + .iter() + .any(|value| value == capability) + } + + /// `true` when this session is bound to the MiniApp's own window rather than a + /// tab the Agent opened. + pub fn is_window_presentation(&self) -> bool { + self.presentation.as_deref() == Some("window") + } + + /// `false` means the MiniApp has not announced an agent bridge yet, so the + /// declared set is unknown rather than empty. The surface runtime answers + /// authoritatively in that case instead of the tool guessing. + pub fn bridge_announced(&self) -> bool { + !self.bridge_capabilities.is_empty() + } + + pub fn to_value(&self) -> Value { + json!({ + "sessionId": self.session_id, + "appId": self.app_id, + "instanceId": self.instance_id, + "surface": { + "kind": match self.is_window_presentation() { + true => "window", + false => "iframe", + }, + "width": self.width, + "height": self.height, + }, + "strictRuntime": self.strict_runtime, + "capabilities": self.capabilities, + "bridgeCapabilities": self.bridge_capabilities, + "bridgeAnnounced": self.bridge_announced(), + "declaredActions": self.declared_actions, + "presentedFrom": self.presented_from, + "targets": self.target_count, + }) + } +} + +/// Process-wide session registry. Sessions are created by `open` and dropped by +/// `close`; they never outlive the surface they were bound to. +#[derive(Debug, Default)] +pub struct SessionRegistry { + sessions: HashMap, + next_id: u64, +} + +impl SessionRegistry { + fn mint_session_id(&mut self) -> String { + let id = format!("miniapp-session-{}", self.next_id.max(1)); + self.next_id = self.next_id.max(1).saturating_add(1); + id + } + + /// Bind a new session to one app. Reopening the same app replaces the older + /// session so the Agent never holds two handles to one surface. + pub fn open(&mut self, surface: MiniAppWebSurfaceInfo) -> &MiniAppWebSession { + self.sessions + .retain(|_, session| session.surface.app_id != surface.app_id); + let session_id = self.mint_session_id(); + let session = MiniAppWebSession { + session_id: session_id.clone(), + surface, + targets: MiniAppWebTargetRegistry::new(), + }; + self.sessions.insert(session_id.clone(), session); + self.sessions + .get(&session_id) + .expect("session was just inserted") + } + + pub fn get(&self, session_id: &str) -> Option<&MiniAppWebSession> { + self.sessions.get(session_id) + } + + pub fn get_mut(&mut self, session_id: &str) -> Option<&mut MiniAppWebSession> { + self.sessions.get_mut(session_id) + } + + pub fn close(&mut self, session_id: &str) -> Option { + self.sessions.remove(session_id) + } + + pub fn list(&self) -> Vec<&MiniAppWebSession> { + self.sessions.values().collect() + } + + pub fn len(&self) -> usize { + self.sessions.len() + } + + pub fn is_empty(&self) -> bool { + self.sessions.is_empty() + } +} + +/// Mint a target handle for a node the host just returned. +pub async fn mint_target( + session_id: &str, + node_key: &str, + summary: String, +) -> Result { + let now_ms = now_ms(); + let mut registry = sessions().lock().await; + let session = registry + .get_mut(session_id) + .ok_or(MiniAppWebErrorCode::UnknownSession)?; + let app_id = session.surface.app_id.clone(); + Ok(session.targets.mint(&app_id, node_key, summary, now_ms)) +} + +/// Resolve a target handle to the host-internal node key. +pub async fn resolve_target( + session_id: &str, + target_id: &str, +) -> Result<(String, String), MiniAppWebErrorCode> { + let now_ms = now_ms(); + let mut registry = sessions().lock().await; + let session = registry + .get_mut(session_id) + .ok_or(MiniAppWebErrorCode::UnknownSession)?; + let app_id = session.surface.app_id.clone(); + let target = session.targets.resolve(&app_id, target_id, now_ms)?; + Ok((target.node_key, target.summary)) +} + +/// Invalidate every earlier handle after a reload or navigation was observed. +pub async fn invalidate_targets(session_id: &str) -> Result<(), MiniAppWebErrorCode> { + let mut registry = sessions().lock().await; + let session = registry + .get_mut(session_id) + .ok_or(MiniAppWebErrorCode::UnknownSession)?; + session.targets.invalidate(); + Ok(()) +} + +/// Host errors are plain strings, but the surface runtime reports typed +/// failures with a leading `CODE:` (for example `STALE_TARGET: frame reloaded`). +/// Recognizing that prefix keeps the tool contract typed without a second +/// protocol on the port. +pub fn classify_surface_error(message: &str) -> MiniAppWebErrorCode { + let Some((code, _rest)) = message.split_once(':') else { + return MiniAppWebErrorCode::HostError; + }; + let code = code.trim(); + [ + MiniAppWebErrorCode::InvalidParams, + MiniAppWebErrorCode::UnknownApp, + MiniAppWebErrorCode::UnknownSession, + MiniAppWebErrorCode::UnknownTarget, + MiniAppWebErrorCode::StaleTarget, + MiniAppWebErrorCode::SurfaceUnavailable, + MiniAppWebErrorCode::CapabilityMissing, + MiniAppWebErrorCode::BridgeUnavailable, + MiniAppWebErrorCode::BridgeTimeout, + MiniAppWebErrorCode::BridgeRejected, + MiniAppWebErrorCode::Timeout, + ] + .into_iter() + .find(|candidate| candidate.as_str() == code) + .unwrap_or(MiniAppWebErrorCode::HostError) +} + +/// Enumerate installed MiniApps through the host port. +/// +/// A host error keeps its typed code so the tool can tell "this runtime cannot +/// enumerate apps" apart from "the enumeration failed". +pub async fn list_installed_apps( +) -> Result, (MiniAppWebErrorCode, String)> { + let host = + miniapp_web_host().map_err(|error| (MiniAppWebErrorCode::SurfaceUnavailable, error))?; + let apps = host + .list_apps() + .await + .map_err(|error| (classify_surface_error(&error), error))?; + let mut apps = apps; + apps.truncate(MINIAPP_WEB_MAX_APPS); + Ok(apps) +} + +/// Forward one typed request to the product surface runtime. +pub async fn execute_surface( + command: MiniAppWebCommand, +) -> Result { + if !SURFACE_METHODS.contains(&command.method.as_str()) { + return Err(( + MiniAppWebErrorCode::InvalidParams, + format!("Unsupported MiniApp surface method: {}", command.method), + )); + } + let host = + miniapp_web_host().map_err(|error| (MiniAppWebErrorCode::SurfaceUnavailable, error))?; + let is_bridge = matches!(command.method.as_str(), "bridge_observe" | "bridge_act"); + let payload = host + .execute(command) + .await + .map_err(|error| (classify_surface_error(&error), error))?; + if is_bridge && validate_bridge_payload_size(&payload).is_err() { + return Err(( + MiniAppWebErrorCode::BridgeRejected, + "BRIDGE_REJECTED: the MiniApp bridge response exceeds the 32 KiB limit".to_string(), + )); + } + Ok(payload) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as u64) + .unwrap_or(0) +} + +/// Parse the `inspect` payload returned by the surface runtime. The tree is +/// re-sanitized here because the frontend is not a trust boundary. +pub fn parse_inspect_payload( + payload: &Value, +) -> Result< + ( + Option, + Option, + Vec, + usize, + bool, + ), + MiniAppWebErrorCode, +> { + let nodes: Vec = payload + .get("tree") + .cloned() + .map(|tree| { + if tree.is_array() { + serde_json::from_value(tree).unwrap_or_default() + } else { + Vec::new() + } + }) + .unwrap_or_default(); + let declared_count = payload + .get("nodeCount") + .and_then(Value::as_u64) + .unwrap_or(nodes.len() as u64) as usize; + let (tree, truncated) = MiniAppWebNode::sanitize_tree(nodes); + let declared_truncated = payload + .get("truncated") + .and_then(Value::as_bool) + .unwrap_or(false); + Ok(( + payload + .get("url") + .and_then(Value::as_str) + .map(str::to_string), + payload + .get("title") + .and_then(Value::as_str) + .map(str::to_string), + tree, + declared_count, + truncated || declared_truncated, + )) +} + +/// Parse the `find` payload: one entry per matching node. +pub fn parse_find_payload(payload: &Value) -> Vec { + let matches: Vec = payload + .get("matches") + .and_then(Value::as_array) + .map(|matches| { + matches + .iter() + .filter_map(|value| serde_json::from_value::(value.clone()).ok()) + .collect() + }) + .unwrap_or_default(); + let (mut matches, _) = MiniAppWebNode::sanitize_tree( + matches + .into_iter() + .map(|mut node| { + node.children.clear(); + node + }) + .collect(), + ); + matches.truncate(20); + matches +} + +/// Bind a freshly opened surface to a new control session. Reopening the same +/// app replaces the older session so one app never has two live handles. +pub async fn open_session(surface: MiniAppWebSurfaceInfo) -> MiniAppWebSessionSnapshot { + sessions().lock().await.open(surface).snapshot() +} + +/// Release a control session. The MiniApp stays open for the user; only the +/// Agent's handles die. +pub async fn close_session(session_id: &str) -> bool { + sessions().lock().await.close(session_id).is_some() +} + +pub async fn sessions_snapshot(session_id: &str) -> Option { + sessions() + .lock() + .await + .get(session_id) + .map(MiniAppWebSession::snapshot) +} + +pub async fn sessions_snapshot_all() -> Vec { + let mut snapshots = sessions() + .lock() + .await + .list() + .into_iter() + .map(MiniAppWebSession::snapshot) + .collect::>(); + snapshots.sort_by(|a, b| a.session_id.cmp(&b.session_id)); + snapshots +} + +/// Store bridge capabilities the MiniApp announced after `open`. The surface +/// runtime is authoritative for what the app declared. +pub async fn refresh_bridge_capabilities(session_id: &str, capabilities: Vec) { + let mut registry = sessions().lock().await; + if let Some(session) = registry.get_mut(session_id) { + session.surface.bridge_capabilities = capabilities; + } +} + +/// Keep the declared action vocabulary in step with the app. +/// +/// The app can re-declare at any time, so a later observation replaces the +/// previous list instead of merging with it. +pub async fn refresh_bridge_actions(session_id: &str, actions: Vec) { + let mut registry = sessions().lock().await; + if let Some(session) = registry.get_mut(session_id) { + session.surface.declared_actions = actions; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecordingHost { + info: MiniAppWebSurfaceInfo, + calls: AtomicUsize, + last_method: RwLock, + response: Option, + } + + impl RecordingHost { + fn new(info: MiniAppWebSurfaceInfo) -> Self { + Self { + info, + calls: AtomicUsize::new(0), + last_method: RwLock::new(String::new()), + response: None, + } + } + } + + #[async_trait] + impl MiniAppWebHost for RecordingHost { + async fn open( + &self, + _request: MiniAppWebOpenRequest, + ) -> Result { + Ok(self.info.clone()) + } + + async fn execute(&self, command: MiniAppWebCommand) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + *write_lock(&self.last_method) = command.method.clone(); + Ok(self + .response + .clone() + .unwrap_or_else(|| json!({ "echo": command.method }))) + } + } + + fn surface(app_id: &str) -> MiniAppWebSurfaceInfo { + MiniAppWebSurfaceInfo { + app_id: app_id.to_string(), + capabilities: MiniAppWebCapability::readable_surface() + .into_iter() + .map(|capability| capability.as_str().to_string()) + .collect(), + ..Default::default() + } + } + + #[tokio::test] + async fn reopening_an_app_replaces_the_previous_session() { + let mut registry = SessionRegistry::default(); + let first = registry.open(surface("demo")).session_id.clone(); + let second = registry.open(surface("demo")).session_id.clone(); + assert_ne!(first, second); + assert!(registry.get(&first).is_none()); + assert_eq!(registry.len(), 1); + assert_eq!(registry.get(&second).unwrap().app_id(), "demo"); + } + + #[tokio::test] + async fn targets_resolve_only_inside_their_session_and_invalidate_on_reload() { + let mut registry = SessionRegistry::default(); + let session_id = registry.open(surface("demo")).session_id.clone(); + let target_id = registry.get_mut(&session_id).unwrap().targets.mint( + "demo", + "n1", + "target-1 button".to_string(), + now_ms(), + ); + assert_eq!( + registry + .get_mut(&session_id) + .unwrap() + .targets + .resolve("demo", &target_id, now_ms()) + .map(|target| target.node_key), + Ok("n1".to_string()) + ); + registry.get_mut(&session_id).unwrap().targets.invalidate(); + assert_eq!( + registry + .get_mut(&session_id) + .unwrap() + .targets + .resolve("demo", &target_id, now_ms()) + .unwrap_err(), + MiniAppWebErrorCode::UnknownTarget + ); + } + + #[tokio::test] + async fn unsupported_surface_methods_are_refused_before_the_host() { + let _guard = HOST_SLOT_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let host = Arc::new(RecordingHost::new(surface("demo"))); + set_miniapp_web_host(host.clone()); + let error = execute_surface(MiniAppWebCommand::new( + "demo", + None, + "evaluate", + Some(json!({"expression": "1+1"})), + )) + .await + .expect_err("arbitrary evaluation must never be forwarded"); + assert_eq!(error.0, MiniAppWebErrorCode::InvalidParams); + assert_eq!(host.calls.load(Ordering::Relaxed), 0); + + let value = execute_surface(MiniAppWebCommand::new("demo", None, "inspect", None)) + .await + .expect("inspect is forwarded"); + assert_eq!(value["echo"], "inspect"); + assert_eq!(*read_lock(&host.last_method), "inspect"); + } + + #[tokio::test] + async fn hosts_that_cannot_enumerate_apps_refuse_instead_of_reporting_none() { + let _guard = HOST_SLOT_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + set_miniapp_web_host(Arc::new(RecordingHost::new(surface("demo")))); + let error = list_installed_apps() + .await + .expect_err("the default enumeration must refuse"); + assert_eq!(error.0, MiniAppWebErrorCode::SurfaceUnavailable); + } + + #[tokio::test] + async fn bridge_responses_are_bounded_at_the_rust_boundary() { + let _guard = HOST_SLOT_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let mut host = RecordingHost::new(surface("demo")); + host.response = Some(json!({"state": "界".repeat(12 * 1024)})); + set_miniapp_web_host(Arc::new(host)); + for method in ["bridge_observe", "bridge_act"] { + let error = execute_surface(MiniAppWebCommand::new("demo", None, method, None)) + .await + .expect_err("oversized bridge output must not reach the model"); + assert_eq!(error.0, MiniAppWebErrorCode::BridgeRejected); + } + // Non-bridge responses, including pixel captures, have their own bounds. + assert!( + execute_surface(MiniAppWebCommand::new("demo", None, "capture", None)) + .await + .is_ok() + ); + } + + #[test] + fn inspect_payload_is_resanitized_and_bounded() { + let payload = json!({ + "url": "miniapp://demo", + "title": "Demo", + "nodeCount": 900, + "truncated": true, + "tree": [{ + "nodeKey": "n1", + "role": "main", + "children": [{"nodeKey": "n2", "role": "button", "name": "Start"}] + }], + }); + let (url, title, tree, count, truncated) = parse_inspect_payload(&payload).unwrap(); + assert_eq!(url.as_deref(), Some("miniapp://demo")); + assert_eq!(title.as_deref(), Some("Demo")); + assert_eq!(count, 900); + assert!(truncated); + assert_eq!(tree[0].children[0].name.as_deref(), Some("Start")); + } + + #[test] + fn find_payload_keeps_flat_matches_only() { + let payload = json!({ + "matches": [ + {"nodeKey": "n1", "role": "button", "name": "Start", "children": [{"nodeKey": "n2"}]}, + {"nodeKey": "n3", "role": "textbox", "name": "City"}, + ], + }); + let matches = parse_find_payload(&payload); + assert_eq!(matches.len(), 2); + assert!(matches[0].children.is_empty()); + assert_eq!(matches[1].node_key.as_deref(), Some("n3")); + } +} diff --git a/src/crates/assembly/core/src/agentic/tools/mod.rs b/src/crates/assembly/core/src/agentic/tools/mod.rs index bd50a6a8c3..8f573e2bc9 100644 --- a/src/crates/assembly/core/src/agentic/tools/mod.rs +++ b/src/crates/assembly/core/src/agentic/tools/mod.rs @@ -19,6 +19,8 @@ mod integer_parser; pub mod manifest_resolver; #[cfg(feature = "tools-miniapp")] pub(crate) mod miniapp_context_runtime; +#[cfg(feature = "tools-miniapp")] +pub mod miniapp_surface; pub mod openbitfun_control_config; pub mod openbitfun_control_host; #[cfg(feature = "tools-pages")] diff --git a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs index 66c47fbffb..13848505ab 100644 --- a/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs +++ b/src/crates/assembly/core/src/agentic/tools/product_runtime/materialization.rs @@ -72,6 +72,7 @@ const PRODUCT_TOOL_REGISTRATION_ORDER: &[&str] = &[ "InitMiniApp", "FinalizeMiniApp", "PublishMiniApp", + "MiniAppBrowser", "FrontendWorkbench", "PublishAppearance", "PageDeploy", @@ -182,6 +183,8 @@ impl StaticToolProviderFactory for ProductConcreteToolFactory { "FinalizeMiniApp" => Some(Arc::new(FinalizeMiniAppTool::new())), #[cfg(feature = "tools-miniapp")] "PublishMiniApp" => Some(Arc::new(PublishMiniAppTool::new())), + #[cfg(feature = "tools-miniapp")] + "MiniAppBrowser" => Some(Arc::new(MiniAppBrowserTool::new())), #[cfg(feature = "tools-creation")] "FrontendWorkbench" => Some(Arc::new(FrontendWorkbenchTool::new())), #[cfg(feature = "tools-miniapp")] diff --git a/src/crates/assembly/core/src/agentic/tools/registry.rs b/src/crates/assembly/core/src/agentic/tools/registry.rs index 4883bd68b6..7fbb19bc78 100644 --- a/src/crates/assembly/core/src/agentic/tools/registry.rs +++ b/src/crates/assembly/core/src/agentic/tools/registry.rs @@ -610,6 +610,7 @@ mod tests { "InitMiniApp", "FinalizeMiniApp", "PublishMiniApp", + "MiniAppBrowser", "FrontendWorkbench", "PublishAppearance", "PageDeploy", @@ -810,6 +811,7 @@ mod tests { "GenerativeUI", "Worktree", "ReviewPlatform", + "MiniAppBrowser", "ControlHub", "Playbook", ], diff --git a/src/crates/assembly/core/src/agentic/tools/tool_context_runtime.rs b/src/crates/assembly/core/src/agentic/tools/tool_context_runtime.rs index 66f19c92fa..e50a744693 100644 --- a/src/crates/assembly/core/src/agentic/tools/tool_context_runtime.rs +++ b/src/crates/assembly/core/src/agentic/tools/tool_context_runtime.rs @@ -155,6 +155,25 @@ impl ToolUseContext { &self.primary_model_facts } + /// Why an image result cannot reach the model in this session, or `None` + /// when images are usable. + /// + /// Every capture-style tool (browser page screenshots, MiniApp frames, …) + /// asks this before spending a round trip: returning base64 the model never + /// sees is not an observation, and it invites endless parameter retries. + pub fn image_observation_unavailable_reason(&self) -> Option<&'static str> { + if !self.primary_model_supports_image_understanding() { + return Some("primary_model_is_text_only"); + } + if !self + .primary_model_facts() + .multimodal_tool_output_supported() + { + return Some("provider_no_multimodal_tool_output"); + } + None + } + pub fn cancellation_token(&self) -> Option<&CancellationToken> { self.runtime_handles.cancellation_token() } diff --git a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json index f9849d0107..9f4c0e28f4 100644 --- a/src/crates/contracts/product-domains/src/generated/product-control-catalog.json +++ b/src/crates/contracts/product-domains/src/generated/product-control-catalog.json @@ -4,7 +4,7 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "de9d03ab0cbb5268869e61d1a3e363c5210d93b9dccb2c8caccc99006074732a", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ { @@ -147,10 +147,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -5765,6 +5765,7 @@ "versions", "agent-authoring", "agent-market-submit", + "agent-operate-app", "publishing" ], "kind": "query", @@ -6711,6 +6712,51 @@ "actionId": "surface.miniapps.open" } }, + { + "id": "feature.miniapps:delegate:agent-operate-app", + "capabilityId": "feature.miniapps", + "itemIds": [ + "agent-operate-app" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "MiniAppBrowser" + ], + "presentationTarget": { + "kind": "action", + "actionId": "surface.miniapps.open" + } + }, { "id": "feature.miniapps:open:publishing", "capabilityId": "feature.miniapps", @@ -22966,6 +23012,25 @@ ] } }, + { + "id": "agent-operate-app", + "titleZh": "由智能体直接操作 Mini App 界面", + "titleEn": "Let an agent operate a Mini App interface directly", + "control": { + "kind": "delegate", + "tools": [ + "MiniAppBrowser" + ], + "workflowZh": [ + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态" + ], + "workflowEn": [ + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" + ] + } + }, { "id": "publishing", "titleZh": "撤回和审查市场发布，并可渲染幻灯片页面", @@ -23336,6 +23401,8 @@ "Let an agent initialize, author, and validate a Mini App", "在用户明确要求时由智能体提交已完成的 Mini App 到市场审核", "Let an agent submit a completed Mini App for market review when explicitly requested", + "由智能体直接操作 Mini App 界面", + "Let an agent operate a Mini App interface directly", "撤回和审查市场发布，并可渲染幻灯片页面", "Withdraw and review market releases, and render slide pages", "打开 Mini Apps", @@ -23361,7 +23428,12 @@ "仅在用户明确要求对外提交时调用 PublishMiniApp，并提供已完成应用、许可与发布说明", "登录、校验或上传返回可恢复状态时，向用户展示具体下一步，不把待授权或待审核状态报告成发布成功", "Call PublishMiniApp only after an explicit request to submit externally, with the completed app, license, and release notes", - "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published" + "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published", + "MiniAppBrowser", + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态", + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" ], "docsUrl": "https://playbook.openbitfun.com/capabilities/feature.miniapps/" }, diff --git a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json index a2a2f05de9..6ea32e67d0 100644 --- a/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json +++ b/src/crates/contracts/product-domains/src/generated/remote-surface-registry.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "digest": "fnv1a64:39cac84a6ae186a5", + "digest": "fnv1a64:63e503f13bf0911c", "retiredCommandPrefixes": [ { "prefix": "lsp_", @@ -4636,6 +4636,18 @@ "kind": "handled" } }, + { + "id": "mark_miniapp_web_surface_ready", + "surface": "tauri_command", + "remoteWorkspace": "WorkspaceAgnostic", + "peer": { + "kind": "controller_local" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" + } + }, { "id": "mark_openbitfun_control_surface_ready", "surface": "tauri_command", @@ -6242,6 +6254,18 @@ "reason": "the controller keeps this command; peer hosts refuse it before dispatch" } }, + { + "id": "report_miniapp_web_result", + "surface": "tauri_command", + "remoteWorkspace": "WorkspaceAgnostic", + "peer": { + "kind": "controller_local" + }, + "cliPeer": { + "kind": "unsupported", + "reason": "the controller keeps this command; peer hosts refuse it before dispatch" + } + }, { "id": "report_openbitfun_control_result", "surface": "tauri_command", diff --git a/src/crates/contracts/product-domains/src/miniapp/bridge_builder.rs b/src/crates/contracts/product-domains/src/miniapp/bridge_builder.rs index 9e9b0fe9a1..1ef89d9adb 100644 --- a/src/crates/contracts/product-domains/src/miniapp/bridge_builder.rs +++ b/src/crates/contracts/product-domains/src/miniapp/bridge_builder.rs @@ -87,6 +87,154 @@ pub fn build_bridge_script( // The script below proactively requests it on startup. let _locale = 'en-US'; + // Agent bridge handlers registered through app.agent.expose(). Null means this + // MiniApp declared no semantic state or actions, so the Agent only gets the + // automatic observation below. + let _agentBridge = null; + + // Automatic observation. Every MiniApp gets this without changing a line of + // app code: what the app draws as text, whether its frame loop is still + // running, and whether dispatched input reached it. That is enough to follow a + // Canvas game that stores state only in pixels. Nothing here reads app + // internals, and all four buffers are bounded so a long-running loop cannot + // grow memory. + const _AUTO_TEXT_LIMIT = 240; + const _AUTO_TEXT_MAX_CHARS = 96; + const _AUTO_CANVAS_LIMIT = 4; + const _autoText = new Map(); + let _autoFrames = 0; + let _autoFrameAt = 0; + let _autoInput = null; + let _autoInputCount = 0; + + const _autoTextAt = (text, x, y) => {{ + if (typeof text !== 'string') return; + const value = text.trim(); + if (!value || value.length > _AUTO_TEXT_MAX_CHARS) return; + const cx = Math.round(Number(x) || 0); + const cy = Math.round(Number(y) || 0); + const key = cx + ',' + cy; + if (!_autoText.has(key) && _autoText.size >= _AUTO_TEXT_LIMIT) {{ + const oldest = _autoText.keys().next().value; + if (oldest !== undefined) _autoText.delete(oldest); + }} + // Same position keeps its slot and only refreshes the text, so a score that + // redraws every frame stays one entry instead of flooding the buffer. + _autoText.set(key, {{ text: value, x: cx, y: cy }}); + }}; + + const _autoWrapDrawText = (method) => {{ + if (typeof CanvasRenderingContext2D === 'undefined') return; + const proto = CanvasRenderingContext2D.prototype; + const original = proto[method]; + if (typeof original !== 'function' || original.__openbitfunAuto === true) return; + const wrapped = function (text, x, y) {{ + try {{ _autoTextAt(text, x, y); }} catch (_error) {{}} + return original.apply(this, arguments); + }}; + wrapped.__openbitfunAuto = true; + proto[method] = wrapped; + }}; + _autoWrapDrawText('fillText'); + _autoWrapDrawText('strokeText'); + + // A frozen loop and a running one look identical in a single screenshot. + const _autoRequestFrame = window.requestAnimationFrame; + if (typeof _autoRequestFrame === 'function') {{ + window.requestAnimationFrame = function (callback) {{ + return _autoRequestFrame.call(window, function (time) {{ + _autoFrames += 1; + _autoFrameAt = Date.now(); + return callback(time); + }}); + }}; + }} + + // Echo of what actually reached the app: an input that never arrived looks the + // same as an input the game ignored, and this separates the two. + ['keydown', 'keyup', 'pointerdown', 'wheel'].forEach((type) => {{ + window.addEventListener(type, (event) => {{ + _autoInputCount += 1; + const key = type === 'wheel' + ? Math.round(event.deltaY || 0) + : (event.key !== undefined ? String(event.key) : (event.button !== undefined ? event.button : null)); + _autoInput = {{ type: type, key: key, at: Date.now() }}; + }}, {{ capture: true, passive: true }}); + }}); + + const _autoCanvases = () => {{ + const canvases = []; + try {{ + document.querySelectorAll('canvas').forEach((canvas) => {{ + if (canvases.length >= _AUTO_CANVAS_LIMIT) return; + const rect = canvas.getBoundingClientRect(); + canvases.push({{ + width: canvas.width, + height: canvas.height, + cssWidth: Math.round(rect.width), + cssHeight: Math.round(rect.height), + }}); + }}); + }} catch (_error) {{}} + return canvases; + }}; + + // Coordinates are in the app's own space; the viewport and canvas sizes let the + // Agent map them onto a screenshot instead of guessing the scale. + const _autoSnapshot = () => ({{ + source: _agentBridge && _agentBridge.observe ? 'declared' : 'auto', + canvasText: Array.from(_autoText.values()).sort((a, b) => a.y - b.y || a.x - b.x), + frames: _autoFrames, + frameAgeMs: _autoFrameAt ? Date.now() - _autoFrameAt : null, + input: _autoInput, + inputCount: _autoInputCount, + viewport: {{ width: window.innerWidth, height: window.innerHeight, dpr: window.devicePixelRatio || 1 }}, + canvases: _autoCanvases(), + }}); + + const _agentBridgeCapabilities = () => {{ + // Automatic observation is always available; declared handlers only add + // labels and semantic actions on top of it. + const capabilities = ['auto-observe']; + if (_agentBridge && _agentBridge.observe) capabilities.push('observe'); + if (_agentBridge && _agentBridge.act) capabilities.push('act'); + return capabilities; + }}; + + // Declared action names are optional and stay bounded: an app that lists the + // action types its `act` handler answers spares the Agent from guessing, and + // an app that lists nothing keeps exactly the older behavior. Names are + // matched case-sensitively because the app decides what they mean. + const _normalizeAgentActions = (values) => {{ + if (!Array.isArray(values)) return []; + const names = []; + for (const value of values) {{ + const raw = typeof value === 'string' + ? value + : (value && typeof value.type === 'string' ? value.type : ''); + const name = typeof raw === 'string' ? raw.trim() : ''; + if (!name || name.length > 48) continue; + if (!/^[\x20-\x7e]+$/.test(name)) continue; + if (names.indexOf(name) !== -1) continue; + names.push(name); + if (names.length >= 32) break; + }} + return names; + }}; + + // Announce on startup and again on every expose(), so a host that mounts later + // still learns the bridge exists. The host pairs this frame by window identity, + // so the app id is only echoed for the protocol's own consistency check. + const _announceAgentBridge = () => {{ + window.parent.postMessage({{ + type: 'openbitfun:miniapp-agent-ready', + schemaVersion: 'agent.v1', + appId: {app_id_esc}, + capabilities: _agentBridgeCapabilities(), + actions: (_agentBridge && Array.isArray(_agentBridge.actions)) ? _agentBridge.actions : [], + }}, '*'); + }}; + const app = {{ get appearanceMode() {{ return _appearanceMode; }}, get locale() {{ return _locale; }}, @@ -172,6 +320,31 @@ pub fn build_bridge_script( cancelStaleRuns: () => _rpc('agent.cancelStaleRuns', {{}}), onEvent: (fn) => app.on('agent:event', fn), offEvent: (fn) => app.off('agent:event', fn), + + // Reverse direction: expose bounded live state to the OpenBitFun Agent and + // accept semantic actions from it. Only what the handlers return is + // visible; the host never inspects app internals. Optional: every frame + // already reports automatic observations (drawn text, loop liveness, input + // echo), and expose() layers labels and exact actions on top of that. + // app.agent.expose({{ + // observe: () => ({{ score, lives, wave, player: {{ x, y }} }}), + // act: (action) => {{ /* e.g. {{ type: 'move', dx }} or {{ type: 'restart' }} */ }}, + // actions: ['move', 'restart'], + // }}); + // `actions` is optional: declare the action types `act` answers so the + // Agent does not have to guess them. It is advisory — the app still + // decides what it accepts, and the host never blocks an undeclared name. + // Needs no manifest permission: everything stays inside this frame. + // Re-calling replaces the previous handlers so a restart can re-anchor. + expose: (handlers) => {{ + const source = handlers || {{}}; + const observe = typeof source.observe === 'function' ? source.observe : null; + const act = typeof source.act === 'function' ? source.act : null; + if (!observe && !act) return false; + _agentBridge = {{ observe, act, actions: _normalizeAgentActions(source.actions) }}; + _announceAgentBridge(); + return true; + }}, }}, // Deck namespace — renders one slide HTML page in a hidden host WebView @@ -313,8 +486,102 @@ pub fn build_bridge_script( }} }}); + // Agent bridge requests: the host asks for state or runs a semantic action the + // MiniApp chose. Replies pair by requestId; a throwing handler becomes a typed + // refusal instead of a hung request. Observation always answers, because the + // automatic half does not depend on the app declaring anything. + window.addEventListener('message', (e) => {{ + const data = e.data; + if (!data || data.type !== 'openbitfun:miniapp-agent-request') return; + const reply = (ok, value, error) => {{ + window.parent.postMessage({{ + type: 'openbitfun:miniapp-agent-response', + requestId: data.requestId, + ok: ok === true, + value: value === undefined ? null : value, + error: error ? String(error) : undefined, + }}, '*'); + }}; + if (data.method === 'observe') {{ + // One envelope for every app: `state` is the app's own declaration (null + // when it declared nothing) and `auto` is what this runtime observed. + const observe = _agentBridge && _agentBridge.observe; + if (typeof observe !== 'function') {{ + reply(true, {{ state: null, auto: _autoSnapshot() }}); + return; + }} + Promise.resolve() + .then(() => observe()) + .then( + (state) => reply(true, {{ + state: state === undefined ? null : state, + auto: _autoSnapshot(), + }}), + (error) => reply(false, null, (error && error.message) || error), + ); + return; + }} + if (data.method === 'act') {{ + const act = _agentBridge && _agentBridge.act; + if (typeof act !== 'function') {{ + reply(false, null, "This MiniApp does not expose 'act'. Input actions and autoplay through key/click remain available."); + return; + }} + Promise.resolve() + .then(() => act(data.action)) + .then( + (value) => reply(true, value), + (error) => reply(false, null, (error && error.message) || error), + ); + return; + }} + if (data.method === 'input') {{ + // Independent MiniApp windows cannot be reached through the Web UI DOM + // router, but the authenticated Shell can still deliver a normal browser + // key event into this frame. This keeps Canvas games controllable without + // granting the host arbitrary JavaScript evaluation or OS-level input. + const action = data.action && typeof data.action === 'object' ? data.action : {{}}; + const rawKey = typeof action.key === 'string' ? action.key : ''; + const trimmedKey = rawKey === ' ' ? ' ' : rawKey.trim(); + const aliases = {{ space: ' ', spacebar: ' ', esc: 'Escape', return: 'Enter', left: 'ArrowLeft', right: 'ArrowRight', up: 'ArrowUp', down: 'ArrowDown' }}; + const key = aliases[trimmedKey.toLowerCase()] || trimmedKey; + if (!key) {{ reply(false, null, 'Input requires a non-empty key'); return; }} + const modifiers = Array.isArray(action.modifiers) ? action.modifiers : []; + const init = {{ + key, + code: key === ' ' ? 'Space' : key, + bubbles: true, + cancelable: true, + altKey: modifiers.some(value => String(value).toLowerCase() === 'alt'), + ctrlKey: modifiers.some(value => String(value).toLowerCase() === 'control'), + metaKey: modifiers.some(value => String(value).toLowerCase() === 'meta'), + shiftKey: modifiers.some(value => String(value).toLowerCase() === 'shift'), + }}; + const holdMs = Math.max(0, Math.min(2000, Number(action.holdMs) || 0)); + const target = document.activeElement && document.activeElement !== document.body + ? document.activeElement + : document.body; + try {{ + target.dispatchEvent(new KeyboardEvent('keydown', init)); + if (!['Alt', 'Control', 'Meta', 'Shift'].includes(key)) target.dispatchEvent(new KeyboardEvent('keypress', init)); + Promise.resolve() + .then(() => holdMs > 0 ? new Promise(resolve => setTimeout(resolve, holdMs)) : undefined) + .then(() => {{ + target.dispatchEvent(new KeyboardEvent('keyup', init)); + reply(true, {{ accepted: true, key, holdMs }}); + }}) + .catch(error => reply(false, null, (error && error.message) || error)); + }} catch (error) {{ + reply(false, null, (error && error.message) || error); + }} + return; + }} + reply(false, null, 'Unsupported bridge method: ' + String(data.method)); + }}); + window.app = app; document.documentElement.setAttribute('data-openbitfun-appearance-mode', _appearanceMode); + _announceAgentBridge(); window.parent.postMessage({{ method: 'openbitfun/request-appearance' }}, '*'); window.parent.postMessage({{ method: 'openbitfun/request-locale' }}, '*'); }})(); @@ -410,3 +677,35 @@ pub fn scroll_boundary_script() -> &'static str { pub fn build_miniapp_default_appearance_css() -> &'static str { include_str!("generated/default_appearance_style.html") } + +#[cfg(test)] +mod tests { + use super::build_bridge_script; + + fn script() -> String { + build_bridge_script("app-1", "/data/app-1", "/workspace", "light", "macos") + } + + #[test] + fn the_announcement_carries_the_declared_actions() { + let script = script(); + // The Agent learns action names from the announcement, so the field has to + // travel with the capabilities an older app already reports. + assert!(script.contains("capabilities: _agentBridgeCapabilities()")); + assert!(script.contains("actions: (_agentBridge && Array.isArray(_agentBridge.actions))")); + } + + #[test] + fn declared_actions_stay_optional_bounded_and_case_sensitive() { + let script = script(); + // An app that declares nothing must behave exactly as before, so the + // normalizer returns an empty list instead of inventing a vocabulary. + assert!(script.contains("const _normalizeAgentActions = (values) => {")); + assert!(script.contains("if (!Array.isArray(values)) return [];")); + assert!(script.contains("if (names.length >= 32) break;")); + assert!(script.contains("if (names.indexOf(name) !== -1) continue;")); + // Action names are the app's own identifiers: no case folding here. + assert!(!script.contains("names.push(name.toLowerCase")); + assert!(script.contains("actions: _normalizeAgentActions(source.actions)")); + } +} diff --git a/src/crates/contracts/product-domains/src/miniapp/mod.rs b/src/crates/contracts/product-domains/src/miniapp/mod.rs index 19e4294811..af0e989e21 100644 --- a/src/crates/contracts/product-domains/src/miniapp/mod.rs +++ b/src/crates/contracts/product-domains/src/miniapp/mod.rs @@ -20,4 +20,5 @@ pub mod runtime; pub mod runtime_facade; pub mod storage; pub mod types; +pub mod web_surface; pub mod worker; diff --git a/src/crates/contracts/product-domains/src/miniapp/web_surface.rs b/src/crates/contracts/product-domains/src/miniapp/web_surface.rs new file mode 100644 index 0000000000..f857075463 --- /dev/null +++ b/src/crates/contracts/product-domains/src/miniapp/web_surface.rs @@ -0,0 +1,1193 @@ +//! Main-Agent to MiniApp Web surface contracts. +//! +//! A MiniApp runs as a Web page inside the product window, so the Agent drives +//! it with BrowserUse semantics (inspect / locate / input / wait) instead of +//! source editing or host paths. This module owns the stable DTOs, limits, and +//! pure helpers shared by the Agent tool, the Desktop adapter, and the Web UI +//! surface runtime: +//! +//! - surface capabilities and session identity, +//! - locator priority and sanitization, +//! - short-lived target handles plus their staleness rules, +//! - the optional MiniApp bridge envelope (`agent-ready` / request / response). +//! +//! Nothing here may carry a `PathManager`, source directory, draft root, or any +//! other controller-local path. + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::collections::HashMap; + +/// Bridge protocol version a MiniApp must echo in `agent-ready`. +pub const MINIAPP_WEB_SCHEMA_VERSION: &str = "agent.v1"; + +/// Upper bound for a single page-structure observation. +pub const MINIAPP_WEB_MAX_NODES: usize = 320; +/// Upper bound for the rendered tree depth. +pub const MINIAPP_WEB_MAX_DEPTH: usize = 12; +/// Upper bound for a node name / label. +pub const MINIAPP_WEB_MAX_NAME_CHARS: usize = 120; +/// Upper bound for a node text or value snippet. +pub const MINIAPP_WEB_MAX_TEXT_CHARS: usize = 400; +/// Upper bound for live target handles in one session. +pub const MINIAPP_WEB_MAX_TARGETS: usize = 320; +/// Upper bound for one `wait` call. +pub const MINIAPP_WEB_MAX_WAIT_MS: u64 = 10_000; +/// Default observation interval for `wait`. +pub const MINIAPP_WEB_DEFAULT_WAIT_MS: u64 = 200; +/// A target handle stops being usable this long after it was minted. +pub const MINIAPP_WEB_TARGET_TTL_MS: u64 = 10 * 60 * 1000; +/// Upper bound for a bridge request or response payload. +pub const MINIAPP_WEB_MAX_BRIDGE_BYTES: usize = 32 * 1024; +/// Upper bound for MiniApp-declared bridge capabilities. +pub const MINIAPP_WEB_MAX_BRIDGE_CAPABILITIES: usize = 8; +/// Upper bound for one bridge round trip. +pub const MINIAPP_WEB_BRIDGE_TIMEOUT_MS: u64 = 5_000; +/// Longest side a captured frame is scaled to before it reaches the model. A +/// Game or Canvas MiniApp has no readable structure, so pixels are the only +/// live observation; the bound keeps one observation readable and cheap. +pub const MINIAPP_WEB_MAX_CAPTURE_EDGE: u32 = 1024; +/// Default longest side for a captured frame. +pub const MINIAPP_WEB_DEFAULT_CAPTURE_EDGE: u32 = 900; +/// Upper bound for one captured image, measured after base64 decoding. +pub const MINIAPP_WEB_MAX_CAPTURE_BYTES: usize = 1_500_000; +/// Upper bound for holding a key down. A game reads held input on its own +/// animation frame, so a key that goes down and up inside one call is invisible +/// to it; this is long enough to move, short enough not to stall a turn. +pub const MINIAPP_WEB_MAX_KEY_HOLD_MS: u64 = 2_000; +/// Upper bound for one `watch_state` window. A turn-paced Agent cannot follow a +/// running app by calling `observe_bridge` in a loop, because every round trip +/// costs a model turn; one bounded sampling window replaces that loop without +/// leaving the app the user is watching. +pub const MINIAPP_WEB_MAX_WATCH_MS: u64 = 120_000; +/// Default sampling window for `watch_state`. +pub const MINIAPP_WEB_DEFAULT_WATCH_MS: u64 = 20_000; +/// Fastest allowed sampling interval. Sampling is a rendered state read, not a +/// frame capture, but it still competes with the app's own loop. +pub const MINIAPP_WEB_MIN_WATCH_POLL_MS: u64 = 250; +/// Default sampling interval. +pub const MINIAPP_WEB_DEFAULT_WATCH_POLL_MS: u64 = 1_000; +/// Upper bound for retained samples in one window. Samples are deduplicated by +/// state, so this only fills up when the app is actually changing. +pub const MINIAPP_WEB_MAX_WATCH_SAMPLES: usize = 120; +/// Upper bound for one retained sample's serialized state. +pub const MINIAPP_WEB_MAX_WATCH_SAMPLE_BYTES: usize = 4 * 1024; + +/// Upper bound for one app-discovery response. A user names a MiniApp, while the +/// surface is addressed by app id, so discovery is how the Agent crosses that +/// gap without reaching host paths. +pub const MINIAPP_WEB_MAX_APPS: usize = 200; +/// Upper bound for a discovered app description. +pub const MINIAPP_WEB_MAX_DESCRIPTION_CHARS: usize = 200; + +pub const BRIDGE_READY_TYPE: &str = "openbitfun:miniapp-agent-ready"; +pub const BRIDGE_REQUEST_TYPE: &str = "openbitfun:miniapp-agent-request"; +pub const BRIDGE_RESPONSE_TYPE: &str = "openbitfun:miniapp-agent-response"; + +/// Web surface capabilities reported by the host for one session. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MiniAppWebCapability { + /// The iframe document is readable, so page structure can be inspected. + Dom, + /// Locators may use accessibility role/name. + Accessibility, + /// The host can dispatch input events into the MiniApp frame. + Input, +} + +impl MiniAppWebCapability { + pub const fn as_str(self) -> &'static str { + match self { + Self::Dom => "dom", + Self::Accessibility => "accessibility", + Self::Input => "input", + } + } + + /// Capabilities granted by a readable (same-origin) MiniApp frame. + pub fn readable_surface() -> Vec { + vec![Self::Dom, Self::Accessibility, Self::Input] + } +} + +/// Stable error taxonomy. Models branch on these codes instead of English text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MiniAppWebErrorCode { + InvalidParams, + UnknownApp, + UnknownSession, + UnknownTarget, + StaleTarget, + SurfaceUnavailable, + CapabilityMissing, + BridgeUnavailable, + BridgeTimeout, + BridgeRejected, + Timeout, + HostError, +} + +impl MiniAppWebErrorCode { + pub const fn as_str(self) -> &'static str { + match self { + Self::InvalidParams => "INVALID_PARAMS", + Self::UnknownApp => "UNKNOWN_APP", + Self::UnknownSession => "UNKNOWN_SESSION", + Self::UnknownTarget => "UNKNOWN_TARGET", + Self::StaleTarget => "STALE_TARGET", + Self::SurfaceUnavailable => "SURFACE_UNAVAILABLE", + Self::CapabilityMissing => "CAPABILITY_MISSING", + Self::BridgeUnavailable => "BRIDGE_UNAVAILABLE", + Self::BridgeTimeout => "BRIDGE_TIMEOUT", + Self::BridgeRejected => "BRIDGE_REJECTED", + Self::Timeout => "TIMEOUT", + Self::HostError => "HOST_ERROR", + } + } +} + +/// One captured frame of a MiniApp surface. +/// +/// A canvas-backed MiniApp (in particular a game) exposes no readable document +/// state, so the only live observation is what the app paints. The surface +/// runtime captures the largest canvas in the frame, scales it within +/// [`MINIAPP_WEB_MAX_CAPTURE_EDGE`], and returns it as an image the model can +/// look at. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MiniAppWebCapture { + pub mime_type: String, + pub data_base64: String, + pub width: u32, + pub height: u32, + /// Which part of the frame was captured, for honest reporting. + pub source: String, + /// Element label or tag the capture came from. + pub label: Option, +} + +impl MiniAppWebCapture { + /// Image types the model side accepts. Anything else is a surface bug. + pub const ALLOWED_MIME_TYPES: &'static [&'static str] = + &["image/png", "image/jpeg", "image/webp"]; + + /// Parse and bound a `capture` payload produced by the surface runtime. The + /// frontend is not a trust boundary, so the image is validated here before + /// it reaches the model. + pub fn parse(payload: &Value) -> Result { + let mime_type = payload + .get("mimeType") + .and_then(Value::as_str) + .map(|value| value.trim().to_ascii_lowercase()) + .filter(|value| Self::ALLOWED_MIME_TYPES.contains(&value.as_str())) + .ok_or(MiniAppWebErrorCode::CapabilityMissing)?; + let data_base64 = payload + .get("dataBase64") + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or(MiniAppWebErrorCode::CapabilityMissing)?; + // Base64 inflates by 4/3, so this rejects an oversized frame without + // decoding it. + if data_base64.len() > MINIAPP_WEB_MAX_CAPTURE_BYTES / 3 * 4 { + return Err(MiniAppWebErrorCode::HostError); + } + let width = payload + .get("width") + .and_then(Value::as_u64) + .unwrap_or(0) + .min(u64::from(MINIAPP_WEB_MAX_CAPTURE_EDGE)) as u32; + let height = payload + .get("height") + .and_then(Value::as_u64) + .unwrap_or(0) + .min(u64::from(MINIAPP_WEB_MAX_CAPTURE_EDGE)) as u32; + let source = payload + .get("source") + .and_then(Value::as_str) + .map(|value| value.chars().take(40).collect::()) + .unwrap_or_else(|| "canvas".to_string()); + let label = payload + .get("label") + .and_then(Value::as_str) + .map(|value| value.chars().take(MINIAPP_WEB_MAX_NAME_CHARS).collect()); + Ok(Self { + mime_type, + data_base64: data_base64.to_string(), + width, + height, + source, + label, + }) + } + + /// Bytes this capture occupies once decoded. + pub fn decoded_len(&self) -> usize { + self.data_base64.len() / 4 * 3 + } +} + +/// Locator axes in resolution priority order. A CSS selector is the last resort +/// because it binds to implementation detail rather than user-visible meaning. +pub const MINIAPP_WEB_LOCATOR_PRIORITY: &[&str] = + &["testId", "roleName", "name", "text", "selector"]; + +/// Element locator. All provided axes must match the same node. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppWebLocator { + pub test_id: Option, + pub role: Option, + pub name: Option, + pub text: Option, + pub selector: Option, + /// Require an exact (not substring) name/text match. + pub exact: Option, +} + +impl MiniAppWebLocator { + /// Trim, drop empty axes, and bound every length before the value reaches + /// the Web UI surface runtime. + pub fn sanitized(self) -> Self { + fn bounded(value: Option, limit: usize) -> Option { + let value = value?; + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + Some(trimmed.chars().take(limit).collect()) + } + + Self { + test_id: bounded(self.test_id, MINIAPP_WEB_MAX_NAME_CHARS), + role: bounded(self.role.map(|role| role.to_ascii_lowercase()), 40), + name: bounded(self.name, MINIAPP_WEB_MAX_NAME_CHARS), + text: bounded(self.text, MINIAPP_WEB_MAX_TEXT_CHARS), + selector: bounded(self.selector, MINIAPP_WEB_MAX_NAME_CHARS), + exact: self.exact, + } + } + + pub fn is_empty(&self) -> bool { + self.test_id.is_none() + && self.role.is_none() + && self.name.is_none() + && self.text.is_none() + && self.selector.is_none() + } + + /// Which axes drive resolution, highest priority first. + pub fn active_axes(&self) -> Vec<&'static str> { + let mut axes = Vec::new(); + if self.test_id.is_some() { + axes.push("testId"); + } + if self.role.is_some() || self.name.is_some() { + axes.push("roleName"); + } + if self.text.is_some() { + axes.push("text"); + } + if self.selector.is_some() { + axes.push("selector"); + } + axes + } + + /// A selector-only locator is a compatibility fallback: the tool warns the + /// model so it prefers stable test ids or accessible names. + pub fn is_selector_fallback(&self) -> bool { + self.selector.is_some() + && self.test_id.is_none() + && self.name.is_none() + && self.text.is_none() + } + + pub fn to_value(&self) -> Value { + json!({ + "testId": self.test_id, + "role": self.role, + "name": self.name, + "text": self.text, + "selector": self.selector, + "exact": self.exact.unwrap_or(false), + }) + } +} + +/// One node of a page-structure observation. `node_key` is a host-internal +/// handle the Web UI mints; it never reaches the model. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppWebNode { + pub node_key: Option, + pub role: Option, + pub name: Option, + pub tag: Option, + pub test_id: Option, + pub value: Option, + pub text: Option, + pub enabled: Option, + pub checked: Option, + pub children: Vec, +} + +impl MiniAppWebNode { + /// Bounded, single-line description used for model-facing summaries. + pub fn summary_line(&self, target_id: &str) -> String { + let mut line = target_id.to_string(); + line.push(' '); + line.push_str(self.role.as_deref().unwrap_or("node")); + if let Some(name) = self.name.as_deref().filter(|name| !name.trim().is_empty()) { + line.push_str(&format!(" {name:?}")); + } + if let Some(test_id) = self.test_id.as_deref().filter(|id| !id.trim().is_empty()) { + line.push_str(&format!(" testId={test_id}")); + } + if self.enabled == Some(false) { + line.push_str(" [disabled]"); + } + if let Some(checked) = self.checked { + line.push_str(if checked { + " [checked]" + } else { + " [unchecked]" + }); + } + if let Some(value) = self + .value + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + line.push_str(&format!(" value={:?}", truncate_chars(value, 60))); + } + line + } + + /// Drop host keys, bound every string, and cut the tree at the contract + /// limits. Returns the sanitized tree and whether anything was dropped. + pub fn sanitize_tree(nodes: Vec) -> (Vec, bool) { + let mut budget = MINIAPP_WEB_MAX_NODES; + let mut truncated = false; + let sanitized = sanitize_level(nodes, 1, &mut budget, &mut truncated); + (sanitized, truncated) + } +} + +fn sanitize_level( + nodes: Vec, + depth: usize, + budget: &mut usize, + truncated: &mut bool, +) -> Vec { + if depth > MINIAPP_WEB_MAX_DEPTH { + *truncated = true; + return Vec::new(); + } + let mut kept = Vec::new(); + for node in nodes { + if *budget == 0 { + *truncated = true; + break; + } + *budget -= 1; + let children = sanitize_level(node.children, depth + 1, budget, truncated); + kept.push(MiniAppWebNode { + node_key: node.node_key, + role: truncate_option(node.role, 40), + name: truncate_option(node.name, MINIAPP_WEB_MAX_NAME_CHARS), + tag: truncate_option(node.tag, 32), + test_id: truncate_option(node.test_id, MINIAPP_WEB_MAX_NAME_CHARS), + value: truncate_option(node.value, MINIAPP_WEB_MAX_TEXT_CHARS), + text: truncate_option(node.text, MINIAPP_WEB_MAX_TEXT_CHARS), + enabled: node.enabled, + checked: node.checked, + children, + }); + } + kept +} + +fn truncate_option(value: Option, limit: usize) -> Option { + let value = value?; + let trimmed = value.trim(); + if trimmed.is_empty() { + return None; + } + Some(truncate_chars(trimmed, limit)) +} + +fn truncate_chars(value: &str, limit: usize) -> String { + if value.chars().count() <= limit { + return value.to_string(); + } + let truncated = value.chars().take(limit).collect::(); + format!("{truncated}…") +} + +/// Surface description returned by `open`. `capabilities` is authoritative for +/// what the Agent may attempt on this session. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppWebSurfaceInfo { + pub instance_id: Option, + pub app_id: String, + pub strict_runtime: bool, + pub width: Option, + pub height: Option, + pub capabilities: Vec, + #[serde(default)] + pub bridge_capabilities: Vec, + /// Action types the app listed for `act`. Empty means it listed none, so the + /// vocabulary is unknown rather than empty: the app decides what it accepts, + /// and this list is advisory, never a gate. + #[serde(default)] + pub declared_actions: Vec, + /// Where the app was presented before this request. A MiniApp already open + /// in its own window is transferred into the tab the user is watching, and + /// saying so is what keeps the move visible instead of surprising. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub presented_from: Option, + /// Which presentation the Agent is bound to. `Some("window")` means the app is + /// running in its own installed window: that presentation can answer the + /// declared agent bridge, but it cannot receive synthetic DOM input or pixels, + /// so the two are not interchangeable and must never be reported as one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub presentation: Option, +} + +/// The presentation kind of a MiniApp running in its own installed window. +pub const MINIAPP_WEB_WINDOW_PRESENTATION: &str = "window"; + +impl MiniAppWebSurfaceInfo { + pub fn has_capability(&self, capability: MiniAppWebCapability) -> bool { + self.capabilities + .iter() + .any(|value| value == capability.as_str()) + } + + pub fn has_bridge_capability(&self, capability: &str) -> bool { + self.bridge_capabilities + .iter() + .any(|value| value == capability) + } + + /// `true` when the Agent controls the MiniApp's own window instead of a tab it + /// opened, which is what makes DOM and pixel actions unavailable. + pub fn is_window_presentation(&self) -> bool { + self.presentation.as_deref() == Some(MINIAPP_WEB_WINDOW_PRESENTATION) + } + + /// A surface that hosts the app in its own window. It answers the declared + /// agent bridge only, so it advertises no DOM, accessibility, or input + /// capability and reports no frame size. + pub fn window(app_id: String, instance_id: String, strict_runtime: bool) -> Self { + Self { + instance_id: Some(instance_id), + app_id, + strict_runtime, + presentation: Some(MINIAPP_WEB_WINDOW_PRESENTATION.to_string()), + ..Default::default() + } + } + + pub fn to_value(&self) -> Value { + json!({ + "appId": self.app_id, + "instanceId": self.instance_id, + "strictRuntime": self.strict_runtime, + "surface": { + "kind": match self.is_window_presentation() { + true => MINIAPP_WEB_WINDOW_PRESENTATION, + false => "iframe", + }, + "width": self.width, + "height": self.height, + }, + "capabilities": self.capabilities, + "bridgeCapabilities": self.bridge_capabilities, + "declaredActions": self.declared_actions, + "presentedFrom": self.presented_from, + }) + } +} + +/// One installed MiniApp, projected for Agent discovery. +/// +/// Discovery is deliberately app-neutral: it answers "which MiniApps exist and +/// how are they addressed", never "what is inside one". +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppWebAppSummary { + pub app_id: String, + pub name: String, + pub description: String, + pub version: u32, + pub updated_at: i64, + /// The app runs in the strict (opaque-origin) profile, so only an app-declared + /// bridge is observable; pixels and DOM are not readable. + pub strict_runtime: bool, +} + +impl MiniAppWebAppSummary { + pub fn to_value(&self) -> Value { + json!({ + "appId": self.app_id, + "name": self.name, + "description": self.description, + "version": self.version, + "updatedAt": self.updated_at, + "strictRuntime": self.strict_runtime, + }) + } + + #[cfg(test)] + fn for_test(app_id: &str, name: &str) -> Self { + Self { + app_id: app_id.to_string(), + name: name.to_string(), + ..Default::default() + } + } +} + +/// Outcome of resolving a requested app against the installed set. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum MiniAppWebAppMatch { + /// Exactly one app matched. + Single(String), + /// Several apps matched; the caller reports candidates instead of guessing. + Ambiguous(Vec), + /// Nothing matched. + Missing, +} + +/// Resolve a user-facing app reference into one installed app id. +/// +/// Priority: exact id, then exact name (case-insensitive), then a name or id +/// substring that matches exactly one app. Anything looser risks driving the +/// wrong app, so ambiguity is reported rather than resolved. +pub fn match_installed_app(apps: &[MiniAppWebAppSummary], query: &str) -> MiniAppWebAppMatch { + let query = query.trim(); + if query.is_empty() { + return MiniAppWebAppMatch::Missing; + } + if let Some(app) = apps.iter().find(|app| app.app_id == query) { + return MiniAppWebAppMatch::Single(app.app_id.clone()); + } + + let needle = query.to_lowercase(); + let mut exact = collect_app_ids(apps.iter().filter(|app| app.name.to_lowercase() == needle)); + if exact.len() == 1 { + return MiniAppWebAppMatch::Single(exact.remove(0)); + } + if exact.len() > 1 { + return MiniAppWebAppMatch::Ambiguous(exact); + } + + let partial = collect_app_ids(apps.iter().filter(|app| { + app.name.to_lowercase().contains(&needle) || app.app_id.to_lowercase().contains(&needle) + })); + match partial.len() { + 0 => MiniAppWebAppMatch::Missing, + 1 => MiniAppWebAppMatch::Single(partial.into_iter().next().unwrap_or_default()), + _ => MiniAppWebAppMatch::Ambiguous(partial), + } +} + +fn collect_app_ids<'a>(apps: impl Iterator) -> Vec { + let mut ids: Vec = Vec::new(); + for app in apps { + if !ids.contains(&app.app_id) { + ids.push(app.app_id.clone()); + } + } + ids.sort(); + ids +} + +/// A minted, short-lived element handle. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MiniAppWebTarget { + pub target_id: String, + pub node_key: String, + pub app_id: String, + pub summary: String, + pub generation: u64, + pub minted_at_ms: u64, +} + +/// Per-MiniApp target handles. Targets are bound to the app that minted them and +/// to one surface generation, so a reload invalidates every earlier handle +/// instead of silently acting on a lookalike node. +#[derive(Debug, Default)] +pub struct MiniAppWebTargetRegistry { + targets: HashMap, + next_id: u64, + generation: u64, +} + +impl MiniAppWebTargetRegistry { + pub fn new() -> Self { + Self { + targets: HashMap::new(), + next_id: 1, + generation: 1, + } + } + + pub fn generation(&self) -> u64 { + self.generation + } + + /// A new document (reload, navigation, re-render) makes every earlier handle + /// stale while keeping the session alive. + pub fn invalidate(&mut self) { + self.targets.clear(); + self.generation = self.generation.saturating_add(1); + } + + pub fn len(&self) -> usize { + self.targets.len() + } + + pub fn is_empty(&self) -> bool { + self.targets.is_empty() + } + + pub fn mint(&mut self, app_id: &str, node_key: &str, summary: String, now_ms: u64) -> String { + self.prune(now_ms); + while self.targets.len() >= MINIAPP_WEB_MAX_TARGETS { + let Some(oldest) = self + .targets + .values() + .min_by_key(|target| target.minted_at_ms) + .map(|target| target.target_id.clone()) + else { + break; + }; + self.targets.remove(&oldest); + } + let target_id = format!("target-{}", self.next_id); + self.next_id = self.next_id.saturating_add(1); + self.targets.insert( + target_id.clone(), + MiniAppWebTarget { + target_id: target_id.clone(), + node_key: node_key.to_string(), + app_id: app_id.to_string(), + summary, + generation: self.generation, + minted_at_ms: now_ms, + }, + ); + target_id + } + + /// Resolve a handle for one app. Unknown ids, expired handles, and handles + /// minted by another MiniApp all fail with a typed error. + pub fn resolve( + &mut self, + app_id: &str, + target_id: &str, + now_ms: u64, + ) -> Result { + self.prune(now_ms); + let target = self + .targets + .get(target_id) + .ok_or(MiniAppWebErrorCode::UnknownTarget)?; + if target.app_id != app_id || target.generation != self.generation { + return Err(MiniAppWebErrorCode::StaleTarget); + } + Ok(target.clone()) + } + + fn prune(&mut self, now_ms: u64) { + self.targets.retain(|_, target| { + now_ms.saturating_sub(target.minted_at_ms) < MINIAPP_WEB_TARGET_TTL_MS + }); + } +} + +/// MiniApp-declared bridge capabilities, as announced in `agent-ready`. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppBridgeReady { + pub app_id: String, + pub schema_version: String, + pub capabilities: Vec, +} + +impl MiniAppBridgeReady { + /// The announcement is untrusted: normalize declared capabilities and + /// reject unknown schema versions instead of guessing. + pub fn validated(mut self) -> Result { + if self.schema_version != MINIAPP_WEB_SCHEMA_VERSION { + return Err(MiniAppWebErrorCode::CapabilityMissing); + } + if self.app_id.trim().is_empty() { + return Err(MiniAppWebErrorCode::InvalidParams); + } + let mut seen = Vec::new(); + for capability in self.capabilities.drain(..) { + let normalized = capability.trim().to_ascii_lowercase(); + if normalized.is_empty() || normalized.len() > 32 || !normalized.is_ascii() { + continue; + } + if !seen.contains(&normalized) { + seen.push(normalized); + } + if seen.len() >= MINIAPP_WEB_MAX_BRIDGE_CAPABILITIES { + break; + } + } + self.capabilities = seen; + Ok(self) + } +} + +/// One bridge response paired by `requestId`. +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase", default)] +pub struct MiniAppBridgeResponse { + pub request_id: String, + pub ok: bool, + pub value: Option, + pub error: Option, +} + +/// Parsed bridge envelope. Unrelated `postMessage` traffic stays `Other`. +#[derive(Clone, Debug, PartialEq)] +pub enum MiniAppBridgeEnvelope { + Ready(MiniAppBridgeReady), + Response(MiniAppBridgeResponse), + Other, +} + +pub fn parse_bridge_envelope(payload: &Value) -> MiniAppBridgeEnvelope { + let message_type = payload.get("type").and_then(Value::as_str).unwrap_or(""); + match message_type { + BRIDGE_READY_TYPE => serde_json::from_value::(payload.clone()) + .map(MiniAppBridgeEnvelope::Ready) + .unwrap_or(MiniAppBridgeEnvelope::Other), + BRIDGE_RESPONSE_TYPE => serde_json::from_value::(payload.clone()) + .map(MiniAppBridgeEnvelope::Response) + .unwrap_or(MiniAppBridgeEnvelope::Other), + _ => MiniAppBridgeEnvelope::Other, + } +} + +/// Build the outbound bridge request. `action` is required for `act` and must be +/// omitted for `observe`; the MiniApp decides what each action means. +pub fn bridge_request_payload( + request_id: &str, + method: &str, + action: Option, +) -> Result { + if request_id.trim().is_empty() { + return Err(MiniAppWebErrorCode::InvalidParams); + } + let mut payload = json!({ + "type": BRIDGE_REQUEST_TYPE, + "schemaVersion": MINIAPP_WEB_SCHEMA_VERSION, + "requestId": request_id, + "method": method, + }); + if let Some(action) = action { + payload["action"] = action; + } + validate_bridge_payload_size(&payload)?; + Ok(payload) +} + +/// Reject oversized bridge payloads before they reach the MiniApp frame. +pub fn validate_bridge_payload_size(payload: &Value) -> Result<(), MiniAppWebErrorCode> { + let size = payload.to_string().len(); + if size > MINIAPP_WEB_MAX_BRIDGE_BYTES { + return Err(MiniAppWebErrorCode::InvalidParams); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(key: &str) -> MiniAppWebNode { + MiniAppWebNode { + node_key: Some(key.to_string()), + role: Some("button".to_string()), + name: Some("Start".to_string()), + tag: Some("button".to_string()), + ..Default::default() + } + } + + #[test] + fn locator_sanitize_drops_blank_axes_and_keeps_priority() { + let locator = MiniAppWebLocator { + test_id: Some(" start-button ".to_string()), + role: Some(" BUTTON ".to_string()), + name: Some(" ".to_string()), + text: None, + selector: None, + exact: None, + } + .sanitized(); + + assert_eq!(locator.test_id.as_deref(), Some("start-button")); + assert_eq!(locator.role.as_deref(), Some("button")); + assert_eq!(locator.name, None); + assert_eq!(locator.active_axes(), vec!["testId", "roleName"]); + assert!(!locator.is_selector_fallback()); + } + + #[test] + fn selector_only_locator_is_a_warned_fallback() { + let locator = MiniAppWebLocator { + selector: Some("#start".to_string()), + ..Default::default() + }; + assert!(locator.is_selector_fallback()); + assert_eq!(locator.active_axes(), vec!["selector"]); + assert!(MiniAppWebLocator::default().is_empty()); + } + + #[test] + fn targets_are_bound_to_one_app_and_generation() { + let mut registry = MiniAppWebTargetRegistry::new(); + let target_id = registry.mint("demo", "n1", "target-1 button".to_string(), 1_000); + assert_eq!(target_id, "target-1"); + assert!(registry.resolve("demo", &target_id, 1_100).is_ok()); + assert_eq!( + registry.resolve("other", &target_id, 1_100), + Err(MiniAppWebErrorCode::StaleTarget) + ); + + registry.invalidate(); + assert_eq!( + registry.resolve("demo", &target_id, 1_200), + Err(MiniAppWebErrorCode::UnknownTarget) + ); + assert_eq!( + registry.resolve("demo", "target-999", 1_200), + Err(MiniAppWebErrorCode::UnknownTarget) + ); + } + + #[test] + fn capture_payload_rejects_unknown_types_and_oversized_frames() { + let capture = MiniAppWebCapture::parse(&json!({ + "mimeType": "IMAGE/PNG", + "dataBase64": "iVBORw0KGgo=", + "width": 900, + "height": 700, + "source": "canvas", + "label": "#game", + })) + .expect("valid capture"); + assert_eq!(capture.mime_type, "image/png"); + assert_eq!(capture.label.as_deref(), Some("#game")); + assert_eq!(capture.decoded_len(), 9); + + assert_eq!( + MiniAppWebCapture::parse(&json!({ + "mimeType": "image/svg+xml", + "dataBase64": "abc", + })), + Err(MiniAppWebErrorCode::CapabilityMissing) + ); + assert_eq!( + MiniAppWebCapture::parse(&json!({ "mimeType": "image/png", "dataBase64": " " })), + Err(MiniAppWebErrorCode::CapabilityMissing) + ); + let oversized = "A".repeat(MINIAPP_WEB_MAX_CAPTURE_BYTES / 3 * 4 + 8); + assert_eq!( + MiniAppWebCapture::parse(&json!({ + "mimeType": "image/jpeg", + "dataBase64": oversized, + })), + Err(MiniAppWebErrorCode::HostError) + ); + // A real frame never exceeds the published edge bound. + let clamped = MiniAppWebCapture::parse(&json!({ + "mimeType": "image/png", + "dataBase64": "iVBORw0KGgo=", + "width": 99_999, + "height": 99_999, + })) + .expect("clamped capture"); + assert_eq!(clamped.width, MINIAPP_WEB_MAX_CAPTURE_EDGE); + assert_eq!(clamped.height, MINIAPP_WEB_MAX_CAPTURE_EDGE); + } + + #[test] + fn app_reference_resolves_by_id_then_exact_name_then_unique_substring() { + let apps = [ + MiniAppWebAppSummary::for_test("7d27c9c1", "雷霆战机"), + MiniAppWebAppSummary::for_test("notes", "Notes"), + MiniAppWebAppSummary::for_test("todo", "Todo list"), + MiniAppWebAppSummary::for_test("todo-pro", "Todo list pro"), + ]; + + assert_eq!( + match_installed_app(&apps, "7d27c9c1"), + MiniAppWebAppMatch::Single("7d27c9c1".to_string()) + ); + assert_eq!( + match_installed_app(&apps, " 雷霆战机 "), + MiniAppWebAppMatch::Single("7d27c9c1".to_string()) + ); + // Case-insensitive exact names still win over substring matches. + assert_eq!( + match_installed_app(&apps, "notes"), + MiniAppWebAppMatch::Single("notes".to_string()) + ); + // An exact id is never widened by a namesake. + assert_eq!( + match_installed_app(&apps, "todo"), + MiniAppWebAppMatch::Single("todo".to_string()) + ); + // A substring that hits two apps is reported, never guessed. + assert_eq!( + match_installed_app(&apps, "tod"), + MiniAppWebAppMatch::Ambiguous(vec!["todo".to_string(), "todo-pro".to_string()]) + ); + assert_eq!( + match_installed_app(&apps, "Todo list pro"), + MiniAppWebAppMatch::Single("todo-pro".to_string()) + ); + assert_eq!( + match_installed_app(&apps, "chess"), + MiniAppWebAppMatch::Missing + ); + assert_eq!( + match_installed_app(&apps, " "), + MiniAppWebAppMatch::Missing + ); + } + + #[test] + fn app_summary_carries_no_host_paths() { + let summary = MiniAppWebAppSummary { + app_id: "demo".to_string(), + name: "Demo".to_string(), + description: "A demo app".to_string(), + version: 3, + updated_at: 42, + strict_runtime: true, + }; + let value = summary.to_value(); + assert_eq!(value["appId"], "demo"); + assert_eq!(value["strictRuntime"], true); + let rendered = value.to_string(); + assert!(!rendered.contains('/'), "discovery must not leak paths"); + } + + #[test] + fn targets_expire_and_stay_bounded() { + let mut registry = MiniAppWebTargetRegistry::new(); + let target_id = registry.mint("demo", "n1", "node".to_string(), 0); + assert_eq!( + registry.resolve("demo", &target_id, MINIAPP_WEB_TARGET_TTL_MS), + Err(MiniAppWebErrorCode::UnknownTarget) + ); + + for index in 0..(MINIAPP_WEB_MAX_TARGETS + 5) { + registry.mint( + "demo", + &format!("n{index}"), + "node".to_string(), + 1_000 + index as u64, + ); + } + assert_eq!(registry.len(), MINIAPP_WEB_MAX_TARGETS); + } + + #[test] + fn tree_sanitize_removes_host_keys_and_caps_nodes() { + let nodes = vec![MiniAppWebNode { + node_key: Some("root".to_string()), + role: Some("main".to_string()), + children: (0..MINIAPP_WEB_MAX_NODES) + .map(|index| node(&format!("n{index}"))) + .collect(), + ..Default::default() + }]; + let (tree, truncated) = MiniAppWebNode::sanitize_tree(nodes); + assert!(truncated); + assert_eq!(tree.len(), 1); + assert!(tree[0].children.len() < MINIAPP_WEB_MAX_NODES); + assert_eq!(tree[0].children[0].node_key.as_deref(), Some("n0")); + } + + #[test] + fn tree_sanitize_stops_at_max_depth() { + let mut deep = node("leaf"); + for index in 0..(MINIAPP_WEB_MAX_DEPTH + 4) { + deep = MiniAppWebNode { + node_key: Some(format!("d{index}")), + children: vec![deep], + ..Default::default() + }; + } + let (tree, truncated) = MiniAppWebNode::sanitize_tree(vec![deep]); + assert!(truncated); + let mut depth = 0; + let mut cursor = tree.first(); + while let Some(current) = cursor { + depth += 1; + cursor = current.children.first(); + } + assert_eq!(depth, MINIAPP_WEB_MAX_DEPTH); + } + + #[test] + fn node_summary_is_compact_and_marks_state() { + let summary = MiniAppWebNode { + enabled: Some(false), + checked: Some(true), + value: Some("abc".to_string()), + ..node("n1") + } + .summary_line("target-7"); + assert!(summary.starts_with("target-7 button \"Start\"")); + assert!(summary.contains("[disabled]")); + assert!(summary.contains("[checked]")); + assert!(summary.contains("value=\"abc\"")); + } + + #[test] + fn bridge_ready_rejects_unknown_schema_and_normalizes_capabilities() { + let ready = MiniAppBridgeReady { + app_id: "demo".to_string(), + schema_version: "agent.v0".to_string(), + capabilities: vec!["observe".to_string()], + }; + assert_eq!( + ready.validated(), + Err(MiniAppWebErrorCode::CapabilityMissing) + ); + + let ready = MiniAppBridgeReady { + app_id: "demo".to_string(), + schema_version: MINIAPP_WEB_SCHEMA_VERSION.to_string(), + capabilities: vec![ + " Observe ".to_string(), + "observe".to_string(), + "ACT".to_string(), + "".to_string(), + "unexpectedly-long-capability-name-over-limit".to_string(), + ], + } + .validated() + .expect("valid announcement"); + assert_eq!(ready.capabilities, vec!["observe", "act"]); + } + + #[test] + fn bridge_envelope_distinguishes_ready_response_and_traffic() { + let ready = json!({ + "type": BRIDGE_READY_TYPE, + "appId": "demo", + "schemaVersion": MINIAPP_WEB_SCHEMA_VERSION, + "capabilities": ["observe"], + }); + assert!(matches!( + parse_bridge_envelope(&ready), + MiniAppBridgeEnvelope::Ready(_) + )); + + let response = json!({ + "type": BRIDGE_RESPONSE_TYPE, + "requestId": "req-1", + "ok": true, + "value": {"phase": "playing"}, + }); + match parse_bridge_envelope(&response) { + MiniAppBridgeEnvelope::Response(response) => { + assert_eq!(response.request_id, "req-1"); + assert_eq!(response.value.unwrap()["phase"], "playing"); + } + other => panic!("unexpected envelope: {other:?}"), + } + + assert_eq!( + parse_bridge_envelope(&json!({"type": "openbitfun:event"})), + MiniAppBridgeEnvelope::Other + ); + } + + #[test] + fn bridge_requests_carry_the_schema_and_reject_oversized_actions() { + let request = bridge_request_payload("req-1", "observe", None).expect("request"); + assert_eq!(request["schemaVersion"], MINIAPP_WEB_SCHEMA_VERSION); + assert!(request.get("action").is_none()); + + let act = bridge_request_payload("req-2", "act", Some(json!({"type": "restart"}))) + .expect("act request"); + assert_eq!(act["action"]["type"], "restart"); + + let oversized = json!({ "type": "noop", "blob": "x".repeat(MINIAPP_WEB_MAX_BRIDGE_BYTES) }); + assert_eq!( + bridge_request_payload("req-3", "act", Some(oversized)), + Err(MiniAppWebErrorCode::InvalidParams) + ); + assert_eq!( + bridge_request_payload(" ", "observe", None), + Err(MiniAppWebErrorCode::InvalidParams) + ); + } + + #[test] + fn surface_info_reports_capabilities_from_the_host() { + let info = MiniAppWebSurfaceInfo { + app_id: "demo".to_string(), + capabilities: vec!["dom".to_string(), "input".to_string()], + bridge_capabilities: vec!["observe".to_string()], + ..Default::default() + }; + assert!(info.has_capability(MiniAppWebCapability::Dom)); + assert!(!info.has_capability(MiniAppWebCapability::Accessibility)); + assert!(info.has_bridge_capability("observe")); + assert!(!info.has_bridge_capability("act")); + assert_eq!(info.to_value()["surface"]["kind"], "iframe"); + } + + #[test] + fn a_window_presentation_is_reported_as_a_different_surface() { + let window = MiniAppWebSurfaceInfo { + app_id: "demo".to_string(), + instance_id: Some("instance-1".to_string()), + presentation: Some("window".to_string()), + ..Default::default() + }; + assert!(window.is_window_presentation()); + let value = window.to_value(); + assert_eq!(value["surface"]["kind"], "window"); + // A window answers only its own bridge, so it advertises no DOM capability. + assert_eq!(value["capabilities"].as_array().map(Vec::len), Some(0)); + } + + #[test] + fn surfaces_written_before_presentations_existed_still_deserialize() { + // Older hosts answered `open` without the field, and an older peer may + // still send exactly this payload. + let legacy = json!({ + "appId": "demo", + "instanceId": "instance-1", + "strictRuntime": true, + "capabilities": ["dom"], + "bridgeCapabilities": [], + }); + let info: MiniAppWebSurfaceInfo = + serde_json::from_value(legacy).expect("legacy surface payload"); + assert_eq!(info.presentation, None); + assert!(!info.is_window_presentation()); + assert_eq!(info.to_value()["surface"]["kind"], "iframe"); + // The field is omitted when a surface never carried one, so an old reader + // sees the same object it always did. + let serialized = serde_json::to_value(&info).expect("reserialized surface"); + assert!(serialized.get("presentation").is_none()); + } +} diff --git a/src/crates/contracts/product-domains/src/remote_surface/table.rs b/src/crates/contracts/product-domains/src/remote_surface/table.rs index 2963eab2ba..0459d0f301 100644 --- a/src/crates/contracts/product-domains/src/remote_surface/table.rs +++ b/src/crates/contracts/product-domains/src/remote_surface/table.rs @@ -473,6 +473,7 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("logout_subscription_account", LocalOnly, Proxied, CLI_NOT_IMPLEMENTED), op("logout_subscription_account_instance", LocalOnly, Proxied, CLI_NOT_IMPLEMENTED), op("manage_dialog_queue", Routed, Proxied, HANDLED), + op("mark_miniapp_web_surface_ready", Agnostic, ControllerLocal, REFUSED), op("mark_openbitfun_control_surface_ready", Agnostic, ControllerLocal, REFUSED), op("mark_openbitfun_control_surface_unready", Agnostic, ControllerLocal, REFUSED), op("market_image_load", Agnostic, ControllerLocal, REFUSED), @@ -608,6 +609,7 @@ pub(super) const OPERATIONS: &[OperationDefinition] = &[ op("report_canvas_runtime_error", Agnostic, Proxied, CLI_NOT_IMPLEMENTED), op("report_canvas_runtime_ready", Agnostic, Proxied, CLI_NOT_IMPLEMENTED), op("report_ide_control_result", Unaudited, ControllerLocal, REFUSED), + op("report_miniapp_web_result", Agnostic, ControllerLocal, REFUSED), op("report_openbitfun_control_result", Agnostic, ControllerLocal, REFUSED), op("reset_agent_profile_config", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), op("reset_assistant_workspace", Unaudited, Proxied, CLI_NOT_IMPLEMENTED), diff --git a/src/crates/contracts/product-domains/tests/miniapp_contracts.rs b/src/crates/contracts/product-domains/tests/miniapp_contracts.rs index b09e578c7a..0ba64750b5 100644 --- a/src/crates/contracts/product-domains/tests/miniapp_contracts.rs +++ b/src/crates/contracts/product-domains/tests/miniapp_contracts.rs @@ -596,6 +596,52 @@ fn miniapp_bridge_exposes_topic_session_lifecycle() { assert!(bridge.contains("chat.completeUserMessage")); } +#[test] +fn miniapp_bridge_exposes_agent_state_bridge() { + let bridge = build_bridge_script("app-1", "/tmp/app", "/tmp/workspace", "dark", "win32"); + + // The MiniApp declares what the Agent may see; the host never inspects + // internals, so a canvas or game becomes observable only through this. + assert!(bridge.contains("expose: (handlers)")); + assert!(bridge.contains("_agentBridge")); + assert!(bridge.contains("openbitfun:miniapp-agent-ready")); + assert!(bridge.contains("openbitfun:miniapp-agent-request")); + assert!(bridge.contains("openbitfun:miniapp-agent-response")); + assert!(bridge.contains("data.method === 'input'")); + assert!(bridge.contains("new KeyboardEvent('keydown', init)")); + assert!(bridge.contains("new KeyboardEvent('keyup', init)")); + // The announcement must echo the real app id so the host can pair the frame. + assert!(bridge.contains("appId: \"app-1\"")); +} + +#[test] +fn miniapp_bridge_observes_an_unchanged_app() { + let bridge = build_bridge_script("app-1", "/tmp/app", "/tmp/workspace", "dark", "win32"); + + // Existing MiniApps must be observable without editing their source, so the + // injected runtime collects a bounded picture of what the app already does. + assert!(bridge.contains("auto-observe")); + assert!(bridge.contains("_autoSnapshot")); + assert!(bridge.contains("fillText")); + assert!(bridge.contains("strokeText")); + assert!(bridge.contains("requestAnimationFrame")); + assert!(bridge.contains("frameAgeMs")); + assert!(bridge.contains("_autoTextAt")); + // One envelope for every app: declared state plus automatic observation. + assert!(bridge.contains("{ state: null, auto: _autoSnapshot() }")); + assert!(bridge.contains("_announceAgentBridge();")); +} + +#[test] +fn miniapp_bridge_never_requires_a_declaration_to_answer_observe() { + let bridge = build_bridge_script("app-1", "/tmp/app", "/tmp/workspace", "dark", "win32"); + + // The old runtime refused every request until the app called expose(); that + // made an untouched game unobservable, which is the bug this contract locks. + assert!(!bridge.contains("This MiniApp does not expose an agent bridge.")); + assert!(bridge.contains("'auto-observe'")); +} + #[test] fn miniapp_permission_policy_preserves_scope_resolution() { let permissions = MiniAppPermissions { diff --git a/src/crates/execution/tool-contracts/src/framework.rs b/src/crates/execution/tool-contracts/src/framework.rs index e22ebdcc1e..a9fa01d294 100644 --- a/src/crates/execution/tool-contracts/src/framework.rs +++ b/src/crates/execution/tool-contracts/src/framework.rs @@ -2349,6 +2349,10 @@ pub fn miniapp_headless_agent_tool_restrictions() -> ToolRuntimeRestrictions { "Playbook", "Playbook is unavailable in MiniApp headless agent runs.", ), + ( + "MiniAppBrowser", + "MiniAppBrowser is unavailable in MiniApp headless agent runs: a MiniApp agent turn must not drive another MiniApp surface.", + ), ( "Cron", "Cron is unavailable in MiniApp headless agent runs.", @@ -2756,6 +2760,7 @@ mod tests { assert!(!restrictions.is_tool_allowed("AskUserQuestion")); assert!(!restrictions.is_tool_allowed("ControlHub")); + assert!(!restrictions.is_tool_allowed("MiniAppBrowser")); assert!(!restrictions.is_tool_allowed("Cron")); assert!(restrictions.is_tool_allowed("AgentSpawn")); assert!(restrictions.is_tool_allowed("WebSearch")); diff --git a/src/crates/execution/tool-provider-groups/src/lib.rs b/src/crates/execution/tool-provider-groups/src/lib.rs index 1caf1f92b8..a9079a3db8 100644 --- a/src/crates/execution/tool-provider-groups/src/lib.rs +++ b/src/crates/execution/tool-provider-groups/src/lib.rs @@ -103,7 +103,9 @@ pub fn tool_feature_group(tool_name: &str) -> Option { "ComputerUse" => Some(ToolPackFeatureGroup::ComputerUse), "view_image" | "analyze_image" => Some(ToolPackFeatureGroup::ImageAnalysis), "GenerativeUI" | "InitMiniApp" | "FinalizeMiniApp" | "PublishMiniApp" - | "PublishAppearance" | "Playbook" => Some(ToolPackFeatureGroup::MiniApp), + | "MiniAppBrowser" | "PublishAppearance" | "Playbook" => { + Some(ToolPackFeatureGroup::MiniApp) + } "PageDeploy" | "PagePublish" => Some(ToolPackFeatureGroup::Pages), "FrontendWorkbench" => Some(ToolPackFeatureGroup::Creation), "CreateCanvas" | "ReadCanvas" | "UpdateCanvas" | "PatchCanvas" => { @@ -261,6 +263,7 @@ const PRODUCT_TOOL_PROVIDER_GROUP_PLAN: &[ToolProviderGroupPlan] = &[ "InitMiniApp", "FinalizeMiniApp", "PublishMiniApp", + "MiniAppBrowser", "PublishAppearance", "Playbook", ], @@ -571,6 +574,7 @@ mod tests { "InitMiniApp", "FinalizeMiniApp", "PublishMiniApp", + "MiniAppBrowser", "PublishAppearance", "Playbook", "PageDeploy", diff --git a/src/shared/interactive-capabilities/catalog.json b/src/shared/interactive-capabilities/catalog.json index 0945556e76..54402e4475 100644 --- a/src/shared/interactive-capabilities/catalog.json +++ b/src/shared/interactive-capabilities/catalog.json @@ -235,11 +235,13 @@ "touch_session_activity" ], "hostBridges": [ + "mark_miniapp_web_surface_ready", "mark_openbitfun_control_surface_ready", "mark_openbitfun_control_surface_unready", "miniapp_shell_call", "peer_host_invoke_complete", "product_control_invoke", + "report_miniapp_web_result", "report_openbitfun_control_result", "report_ide_control_result", "webdriver_bridge_result" @@ -3274,6 +3276,29 @@ "command:miniapp_market_submit_installed" ] }, + { + "id": "agent-operate-app", + "titleZh": "由智能体直接操作 Mini App 界面", + "titleEn": "Let an agent operate a Mini App interface directly", + "control": { + "kind": "delegate", + "tools": [ + "MiniAppBrowser" + ], + "workflowZh": [ + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态" + ], + "workflowEn": [ + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" + ] + }, + "evidence": [ + "source:src/crates/assembly/core/src/agentic/tools/implementations/miniapp_browser/mod.rs#MiniAppBrowser", + "source:src/crates/contracts/product-domains/src/miniapp/web_surface.rs#MiniAppWebLocator" + ] + }, { "id": "publishing", "titleZh": "撤回和审查市场发布，并可渲染幻灯片页面", @@ -9817,6 +9842,9 @@ "miniapp_market": { "capabilityId": "feature.miniapps" }, + "miniapp_web_host": { + "capabilityId": "feature.miniapps" + }, "market_image": { "capabilityId": null }, diff --git a/src/web-ui/src/app/global-search/generated/interactive-capabilities.json b/src/web-ui/src/app/global-search/generated/interactive-capabilities.json index 5fde780727..9a76c0e6ad 100644 --- a/src/web-ui/src/app/global-search/generated/interactive-capabilities.json +++ b/src/web-ui/src/app/global-search/generated/interactive-capabilities.json @@ -4,7 +4,7 @@ "title": "OpenBitFun Playbook", "origin": "https://playbook.openbitfun.com", "source": "src/shared/interactive-capabilities/catalog.json", - "digest": "65f1a11c6d994c382da25f2b9ad995ad0f443a5d352c2eafa10a8b9bcbf849e5", + "digest": "de9d03ab0cbb5268869e61d1a3e363c5210d93b9dccb2c8caccc99006074732a", "ownerDigest": "4e1ace270088acecbd333d2c8aced0547dddb0fd6bae778bec24f2b0fb48ea91", "searchAcceptance": [ { @@ -147,10 +147,10 @@ "features": 23, "settings": 21, "userFacing": 44, - "documentedItems": 333, + "documentedItems": 334, "controlCoverage": { "direct": 52, - "delegated": 65, + "delegated": 66, "interactive": 216, "unsupported": 0 } @@ -5765,6 +5765,7 @@ "versions", "agent-authoring", "agent-market-submit", + "agent-operate-app", "publishing" ], "kind": "query", @@ -6711,6 +6712,51 @@ "actionId": "surface.miniapps.open" } }, + { + "id": "feature.miniapps:delegate:agent-operate-app", + "capabilityId": "feature.miniapps", + "itemIds": [ + "agent-operate-app" + ], + "kind": "delegate", + "risk": "execute", + "executionHost": "workspaceHost", + "availability": { + "desktop": { + "available": true + }, + "cli": { + "available": true + }, + "peer": { + "available": true, + "requiredCapabilities": [ + "product_control_v1" + ] + }, + "remoteControl": { + "available": true + }, + "detachedDispatch": { + "available": true + } + }, + "inputSchema": { + "type": "object", + "additionalProperties": true + }, + "outputSchema": { + "type": "object", + "additionalProperties": true + }, + "delegateTools": [ + "MiniAppBrowser" + ], + "presentationTarget": { + "kind": "action", + "actionId": "surface.miniapps.open" + } + }, { "id": "feature.miniapps:open:publishing", "capabilityId": "feature.miniapps", @@ -22950,6 +22996,25 @@ ] } }, + { + "id": "agent-operate-app", + "titleZh": "由智能体直接操作 Mini App 界面", + "titleEn": "Let an agent operate a Mini App interface directly", + "control": { + "kind": "delegate", + "tools": [ + "MiniAppBrowser" + ], + "workflowZh": [ + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态" + ], + "workflowEn": [ + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" + ] + } + }, { "id": "publishing", "titleZh": "撤回和审查市场发布，并可渲染幻灯片页面", @@ -23295,6 +23360,8 @@ "Let an agent initialize, author, and validate a Mini App", "在用户明确要求时由智能体提交已完成的 Mini App 到市场审核", "Let an agent submit a completed Mini App for market review when explicitly requested", + "由智能体直接操作 Mini App 界面", + "Let an agent operate a Mini App interface directly", "撤回和审查市场发布，并可渲染幻灯片页面", "Withdraw and review market releases, and render slide pages", "打开 Mini Apps", @@ -23320,7 +23387,12 @@ "仅在用户明确要求对外提交时调用 PublishMiniApp，并提供已完成应用、许可与发布说明", "登录、校验或上传返回可恢复状态时，向用户展示具体下一步，不把待授权或待审核状态报告成发布成功", "Call PublishMiniApp only after an explicit request to submit externally, with the completed app, license, and release notes", - "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published" + "When authentication, validation, or upload returns a resumable state, show the exact next step and never report pending authorization or review as published", + "MiniAppBrowser", + "用户要求在 Mini App 内操作时，先用 MiniAppBrowser 的 open 打开应用并 inspect 读取可读结构，再按返回的 targetId 执行 find、read、click、type、key、scroll", + "只有应用声明了 bridge 能力时才使用 observe_bridge 与 act_bridge；strict runtime 或应用在独立窗口运行时如实报告界面不可读的原因，不猜测应用内部状态", + "When the user asks for interaction inside a Mini App, open it with MiniAppBrowser and inspect the readable structure, then act on the returned target IDs with find, read, click, type, key, and scroll", + "Use observe_bridge and act_bridge only when the app announces bridge capabilities; when a strict runtime or a separately presented window makes the surface unreadable, report the exact reason instead of guessing the app's internal state" ], "docsUrl": "https://playbook.openbitfun.com/capabilities/feature.miniapps/" }, diff --git a/src/web-ui/src/app/scenes/miniapps/agent/MiniAppAgentSurface.tsx b/src/web-ui/src/app/scenes/miniapps/agent/MiniAppAgentSurface.tsx new file mode 100644 index 0000000000..a3c92d389d --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/MiniAppAgentSurface.tsx @@ -0,0 +1,10 @@ +/** + * Headless host for the Agent-controlled MiniApp surface. Mount exactly once in + * the main window, next to `MiniAppPresence`. + */ +import { useMiniAppAgentSurface } from './useMiniAppAgentSurface'; + +export function MiniAppAgentSurface() { + useMiniAppAgentSurface(); + return null; +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.test.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.test.ts new file mode 100644 index 0000000000..fb170976bb --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.test.ts @@ -0,0 +1,212 @@ +/** + * @vitest-environment jsdom + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { MiniAppAgentBridge } from './miniAppAgentBridge'; +import { + MINIAPP_AGENT_BRIDGE_READY_TYPE, + MINIAPP_AGENT_BRIDGE_RESPONSE_TYPE, + MINIAPP_AGENT_SCHEMA_VERSION, + MiniAppAgentSurfaceError, +} from './miniAppAgentProtocol'; + +function createBridge() { + const posts: Record[] = []; + const contentWindow = { + postMessage: (payload: Record) => posts.push(payload), + } as unknown as Window; + const iframe = { contentWindow } as unknown as HTMLIFrameElement; + const bridge = new MiniAppAgentBridge({ appId: 'demo', iframe }); + bridge.start(); + return { bridge, contentWindow, posts }; +} + +function announce(source: unknown, data: Record) { + const event = new MessageEvent('message', { data }); + Object.defineProperty(event, 'source', { value: source }); + window.dispatchEvent(event); +} + +function respond(source: unknown, data: Record) { + announce(source, { type: MINIAPP_AGENT_BRIDGE_RESPONSE_TYPE, ...data }); +} + +function ready(capabilities: string[] = ['observe', 'act'], appId = 'demo') { + return { + type: MINIAPP_AGENT_BRIDGE_READY_TYPE, + appId, + schemaVersion: MINIAPP_AGENT_SCHEMA_VERSION, + capabilities, + }; +} + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + expect(error).toBeInstanceOf(MiniAppAgentSurfaceError); + return error as MiniAppAgentSurfaceError; + } + throw new Error('expected the bridge call to fail'); +} + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('MiniApp agent bridge', () => { + it('only accepts an announcement from its own frame and app', async () => { + const { bridge, contentWindow } = createBridge(); + announce({}, ready()); // a different source + announce(contentWindow, ready(['observe'], 'someone-else')); + expect(bridge.announced).toBe(false); + + announce(contentWindow, ready([' Observe ', 'observe', 'ACT', 'not ascii ✓'])); + expect(bridge.announced).toBe(true); + expect(bridge.capabilities).toEqual(['observe', 'act']); + }); + + it('rejects an unknown schema version instead of guessing capabilities', () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, { ...ready(), schemaVersion: 'agent.v0' }); + expect(bridge.announced).toBe(false); + expect(bridge.capabilities).toEqual([]); + }); + + it('pairs a response by requestId and forwards the action payload', async () => { + const { bridge, contentWindow, posts } = createBridge(); + announce(contentWindow, ready()); + const pending = bridge.act('req-1', { type: 'restart' }); + expect(posts[0]).toMatchObject({ + requestId: 'req-1', + method: 'act', + schemaVersion: MINIAPP_AGENT_SCHEMA_VERSION, + action: { type: 'restart' }, + }); + + respond(contentWindow, { requestId: 'req-1', ok: true, value: { running: true } }); + await expect(pending).resolves.toEqual({ + result: { running: true }, + capabilities: ['observe', 'act'], + actions: [], + }); + }); + + it('rejects oversized UTF-8 and non-JSON responses without losing the next request', async () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, ready()); + const cyclic: Record = {}; + cyclic.self = cyclic; + for (const value of ['x'.repeat(33 * 1024), '界'.repeat(12 * 1024), cyclic]) { + const pending = bridge.observe('oversized'); + respond(contentWindow, { requestId: 'oversized', ok: true, value }); + expect((await rejection(pending)).code).toBe('BRIDGE_REJECTED'); + } + const next = bridge.observe('next'); + respond(contentWindow, { requestId: 'next', ok: true, value: { state: { count: 1 } } }); + await expect(next).resolves.toMatchObject({ state: { count: 1 } }); + bridge.stop(); + }); + + it('applies the byte bound to outgoing multibyte actions as well', async () => { + const { bridge, contentWindow, posts } = createBridge(); + announce(contentWindow, ready()); + expect((await rejection(bridge.act('large', { type: 'write', text: '界'.repeat(12 * 1024) }))).code) + .toBe('INVALID_PARAMS'); + expect(posts).toHaveLength(0); + bridge.stop(); + }); + + it('fails loudly when the app declares no bridge or lacks the capability', async () => { + const { bridge, contentWindow } = createBridge(); + expect((await rejection(bridge.observe('req-1'))).code).toBe('BRIDGE_UNAVAILABLE'); + + announce(contentWindow, ready(['observe'])); + expect((await rejection(bridge.act('req-2', { type: 'restart' }))).code).toBe('BRIDGE_UNAVAILABLE'); + }); + + it('observes an app that declares nothing, through automatic observation', async () => { + const { bridge, contentWindow } = createBridge(); + // The injected runtime always announces auto-observe, so an untouched app is + // observable without editing its source. + announce(contentWindow, ready(['auto-observe'])); + const pending = bridge.observe('req-1'); + respond(contentWindow, { + requestId: 'req-1', + ok: true, + value: { state: null, auto: { source: 'auto', canvasText: [{ text: 'SCORE 1200' }], frames: 9 } }, + }); + + await expect(pending).resolves.toEqual({ + state: null, + auto: { source: 'auto', canvasText: [{ text: 'SCORE 1200' }], frames: 9 }, + capabilities: ['auto-observe'], + actions: [], + }); + // Automatic observation never implies semantic actions. + expect((await rejection(bridge.act('req-2', { type: 'restart' }))).code).toBe('BRIDGE_UNAVAILABLE'); + }); + + it('reports the action vocabulary the app declared, case preserved', async () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, { + ...ready(['observe', 'act']), + // Apps name their own actions, so cased names must survive; malformed, + // duplicate and over-long entries are dropped instead of forwarded. + actions: [' start ', 'Autoplay', 'start', 42, { type: 'pause' }, 'x'.repeat(60), 'не ascii'], + }); + expect(bridge.actions).toEqual(['start', 'Autoplay', 'pause']); + + const pending = bridge.act('req-1', { type: 'autoplay', maxSteps: 200 }); + respond(contentWindow, { requestId: 'req-1', ok: true, value: { running: true } }); + await expect(pending).resolves.toEqual({ + result: { running: true }, + capabilities: ['observe', 'act'], + actions: ['start', 'Autoplay', 'pause'], + }); + + // An app that declares nothing reports an empty list, which means unknown + // rather than "accepts nothing". + const fresh = createBridge(); + announce(fresh.contentWindow, ready(['observe', 'act'])); + expect(fresh.bridge.actions).toEqual([]); + }); + + it('keeps declared state authoritative while still reporting automatic observation', async () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, ready(['auto-observe', 'observe', 'act'])); + const pending = bridge.observe('req-1'); + respond(contentWindow, { + requestId: 'req-1', + ok: true, + value: { state: { score: 1300 }, auto: { source: 'declared', frames: 4 } }, + }); + + await expect(pending).resolves.toMatchObject({ + state: { score: 1300 }, + auto: { source: 'declared' }, + }); + }); + + it('times out instead of waiting forever, and surfaces app rejections', async () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, ready()); + expect((await rejection(bridge.observe('req-1', 5))).code).toBe('BRIDGE_TIMEOUT'); + + const rejected = bridge.observe('req-2'); + respond(contentWindow, { requestId: 'req-2', ok: false, error: 'the game is paused' }); + const error = await rejection(rejected); + expect(error.code).toBe('BRIDGE_REJECTED'); + expect(error.message).toContain('the game is paused'); + }); + + it('rejects pending calls when the surface detaches', async () => { + const { bridge, contentWindow } = createBridge(); + announce(contentWindow, ready()); + const pending = bridge.observe('req-1'); + bridge.stop(); + expect((await rejection(pending)).code).toBe('SURFACE_UNAVAILABLE'); + }); +}); diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.ts new file mode 100644 index 0000000000..297f9ec72e --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentBridge.ts @@ -0,0 +1,263 @@ +/** + * Optional MiniApp agent bridge client. + * + * A MiniApp that wants to expose bounded JSON state (Canvas apps, games, custom + * renderers) announces `openbitfun:miniapp-agent-ready` from its frame and + * answers `openbitfun:miniapp-agent-request` messages. The host never inspects + * a MiniApp's internals on its own: capability and payload shape are the app's + * declaration, and the host only pairs requests by `requestId` with a timeout. + */ + +import { + MINIAPP_AGENT_BRIDGE_READY_TYPE, + MINIAPP_AGENT_BRIDGE_REQUEST_TYPE, + MINIAPP_AGENT_BRIDGE_RESPONSE_TYPE, + MINIAPP_AGENT_BRIDGE_TIMEOUT_MS, + MINIAPP_AGENT_MAX_ACTION_NAME_CHARS, + MINIAPP_AGENT_MAX_BRIDGE_ACTIONS, + MINIAPP_AGENT_MAX_BRIDGE_CAPABILITIES, + MINIAPP_AGENT_SCHEMA_VERSION, + surfaceError, + assertMiniAppAgentBridgePayload, + type MiniAppAgentBridgeAnnouncement, + type MiniAppAgentBridgeResponse, +} from './miniAppAgentProtocol'; + +export interface MiniAppAgentBridgeSurface { + appId: string; + iframe: HTMLIFrameElement; +} + +export type MiniAppAgentBridgeMethod = 'observe' | 'act'; + +interface PendingRequest { + resolve: (response: MiniAppAgentBridgeResponse) => void; + reject: (error: unknown) => void; + timer: ReturnType; +} + +function normalizeCapabilities(values: unknown): string[] { + if (!Array.isArray(values)) return []; + const normalized: string[] = []; + for (const value of values) { + if (typeof value !== 'string') continue; + const capability = value.trim().toLowerCase(); + if (!capability || capability.length > 32) continue; + if (!/^[\x20-\x7e]+$/.test(capability)) continue; + if (normalized.includes(capability)) continue; + normalized.push(capability); + if (normalized.length >= MINIAPP_AGENT_MAX_BRIDGE_CAPABILITIES) break; + } + return normalized; +} + +/** + * Declared action names stay case-sensitive: the app decides what they mean, so + * normalizing case here would invent a name the app does not answer. + */ +function normalizeActions(values: unknown): string[] { + if (!Array.isArray(values)) return []; + const normalized: string[] = []; + for (const value of values) { + const raw = typeof value === 'string' + ? value + : (value && typeof value === 'object' && typeof (value as { type?: unknown }).type === 'string' + ? (value as { type: string }).type + : ''); + const action = raw.trim(); + if (!action || action.length > MINIAPP_AGENT_MAX_ACTION_NAME_CHARS) continue; + if (!/^[\x20-\x7e]+$/.test(action)) continue; + if (normalized.includes(action)) continue; + normalized.push(action); + if (normalized.length >= MINIAPP_AGENT_MAX_BRIDGE_ACTIONS) break; + } + return normalized; +} + +export class MiniAppAgentBridge { + private announcement: MiniAppAgentBridgeAnnouncement | null = null; + private readonly pending = new Map(); + private listening = false; + private readonly handler = (event: MessageEvent) => this.onMessage(event); + + constructor(private readonly surface: MiniAppAgentBridgeSurface) {} + + start(): void { + if (this.listening) return; + this.listening = true; + window.addEventListener('message', this.handler); + } + + stop(): void { + if (this.listening) { + window.removeEventListener('message', this.handler); + this.listening = false; + } + this.rejectPending(surfaceError('SURFACE_UNAVAILABLE', 'The MiniApp surface detached.')); + } + + get announced(): boolean { + return this.announcement !== null; + } + + get capabilities(): string[] { + return this.announcement ? [...this.announcement.capabilities] : []; + } + + /** + * Action types the app declared for `act`. Empty means it declared nothing, + * so the names are unknown rather than nonexistent. + */ + get actions(): string[] { + return this.announcement ? [...this.announcement.actions] : []; + } + + async observe(requestId: string, timeoutMs = MINIAPP_AGENT_BRIDGE_TIMEOUT_MS): Promise> { + const value = await this.request(requestId, 'observe', undefined, timeoutMs); + // The injected runtime answers with one envelope: the app's own declaration + // under `state` and its automatic observation under `auto`. + const envelope = value && typeof value === 'object' ? value as Record : {}; + return { + state: envelope.state ?? null, + auto: envelope.auto ?? null, + capabilities: this.capabilities, + actions: this.actions, + }; + } + + async act( + requestId: string, + action: unknown, + timeoutMs = MINIAPP_AGENT_BRIDGE_TIMEOUT_MS, + ): Promise> { + const value = await this.request(requestId, 'act', action, timeoutMs); + return { result: value ?? null, capabilities: this.capabilities, actions: this.actions }; + } + + private async request( + requestId: string, + method: MiniAppAgentBridgeMethod, + action: unknown, + timeoutMs: number, + ): Promise { + if (!requestId.trim()) { + throw surfaceError('INVALID_PARAMS', 'The bridge request is missing its requestId.'); + } + if (!this.announced) { + throw surfaceError( + 'BRIDGE_UNAVAILABLE', + "This MiniApp did not announce an agent bridge, so its internal state is not observable. DOM actions require a readable frame instead.", + ); + } + // `auto-observe` ships with the injected runtime, so observation works for an + // app that declares nothing; only `act` needs an explicit declaration. + const declared = this.capabilities.includes(method) + || (method === 'observe' && this.capabilities.includes('auto-observe')); + if (!declared) { + throw surfaceError( + 'BRIDGE_UNAVAILABLE', + `This MiniApp does not declare the '${method}' agent bridge capability.`, + ); + } + const target = this.surface.iframe.contentWindow; + if (!target) { + throw surfaceError('SURFACE_UNAVAILABLE', 'The MiniApp frame is not mounted.'); + } + const payload: Record = { + type: MINIAPP_AGENT_BRIDGE_REQUEST_TYPE, + schemaVersion: MINIAPP_AGENT_SCHEMA_VERSION, + requestId, + method, + }; + if (method === 'act') payload.action = action; + assertMiniAppAgentBridgePayload(payload); + + const response = await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(requestId); + reject(surfaceError( + 'BRIDGE_TIMEOUT', + `The MiniApp did not answer the '${method}' bridge request in ${timeoutMs}ms.`, + )); + }, timeoutMs); + this.pending.set(requestId, { resolve, reject, timer }); + try { + target.postMessage(payload, '*'); + } catch (error) { + clearTimeout(timer); + this.pending.delete(requestId); + reject(surfaceError('HOST_ERROR', `Failed to deliver the bridge request: ${String(error)}`)); + } + }); + + if (!response.ok) { + throw surfaceError( + 'BRIDGE_REJECTED', + response.error?.trim() || 'The MiniApp rejected the bridge action.', + ); + } + return response.value; + } + + private onMessage(event: MessageEvent): void { + // Opaque-origin frames cannot be verified by origin, so the frame's own + // window is the identity check. + if (event.source !== this.surface.iframe.contentWindow) return; + const data = event.data as Record | null; + if (!data || typeof data !== 'object') return; + if (data.type === MINIAPP_AGENT_BRIDGE_READY_TYPE) { + this.acceptAnnouncement(data); + return; + } + if (data.type === MINIAPP_AGENT_BRIDGE_RESPONSE_TYPE) { + this.acceptResponse(data); + } + } + + private acceptAnnouncement(data: Record): void { + const appId = typeof data.appId === 'string' ? data.appId.trim() : ''; + // A frame may only speak for the app it is hosting. + if (!appId || appId !== this.surface.appId) return; + const schemaVersion = typeof data.schemaVersion === 'string' ? data.schemaVersion.trim() : ''; + if (schemaVersion !== MINIAPP_AGENT_SCHEMA_VERSION) { + // Keep `announced` false: an unknown schema is not a usable bridge. + this.announcement = null; + return; + } + this.announcement = { + appId, + schemaVersion, + capabilities: normalizeCapabilities(data.capabilities), + actions: normalizeActions(data.actions), + }; + } + + private acceptResponse(data: Record): void { + const requestId = typeof data.requestId === 'string' ? data.requestId : ''; + if (!requestId) return; + const pending = this.pending.get(requestId); + if (!pending) return; + this.pending.delete(requestId); + clearTimeout(pending.timer); + try { + assertMiniAppAgentBridgePayload(data, 'BRIDGE_REJECTED'); + } catch (error) { + pending.reject(error); + return; + } + pending.resolve({ + requestId, + ok: data.ok === true, + value: data.value, + error: typeof data.error === 'string' ? data.error : undefined, + }); + } + + private rejectPending(error: unknown): void { + for (const [requestId, pending] of this.pending) { + clearTimeout(pending.timer); + this.pending.delete(requestId); + pending.reject(error); + } + } +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.test.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.test.ts new file mode 100644 index 0000000000..d4acb66245 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.test.ts @@ -0,0 +1,127 @@ +/** + * @vitest-environment jsdom + */ + +import { describe, expect, it } from 'vitest'; + +import { captureSurface } from './miniAppAgentCapture'; +import type { MiniAppAgentDomSurface } from './miniAppAgentDomRuntime'; +import { MiniAppAgentSurfaceError } from './miniAppAgentProtocol'; + +function createSurface(html: string): MiniAppAgentDomSurface { + const doc = document.implementation.createHTMLDocument('MiniApp'); + doc.body.innerHTML = html; + const iframe = { contentDocument: doc } as unknown as HTMLIFrameElement; + return { appId: 'demo', iframe }; +} + +/** + * jsdom has no 2D canvas backend, so the capture path runs against a stub that + * records what the runtime drew and returns a bounded data URL. + */ +function stubCanvas( + payload: number | ((width: number) => number), +): { drawn: { width: number; height: number }[]; restore: () => void } { + const prototype = HTMLCanvasElement.prototype; + const contextDescriptor = Object.getOwnPropertyDescriptor(prototype, 'getContext'); + const dataUrlDescriptor = Object.getOwnPropertyDescriptor(prototype, 'toDataURL'); + const drawn: { width: number; height: number }[] = []; + + Object.defineProperty(prototype, 'getContext', { + configurable: true, + value: () => ({ + drawImage: (_source: unknown, _x: number, _y: number, width: number, height: number) => { + drawn.push({ width, height }); + }, + }), + }); + Object.defineProperty(prototype, 'toDataURL', { + configurable: true, + value: (type: string) => { + const width = drawn[drawn.length - 1]?.width ?? 0; + const size = typeof payload === 'function' ? payload(width) : payload; + return `data:${type};base64,${'A'.repeat(Math.max(0, size))}`; + }, + }); + + return { + drawn, + restore: () => { + if (contextDescriptor) Object.defineProperty(prototype, 'getContext', contextDescriptor); + else delete (prototype as unknown as Record).getContext; + if (dataUrlDescriptor) Object.defineProperty(prototype, 'toDataURL', dataUrlDescriptor); + else delete (prototype as unknown as Record).toDataURL; + }, + }; +} + +describe('MiniApp agent capture', () => { + it('captures the largest canvas, scaled into the observation bound', async () => { + const stub = stubCanvas(64); + try { + const surface = createSurface(` + + + `); + const capture = await captureSurface(surface, 900); + + expect(capture.label).toBe('#game'); + expect(capture.source).toBe('canvas'); + expect(capture.width).toBe(900); + expect(capture.height).toBe(675); + // A scaled-down frame ships as JPEG: smaller payload, same observation. + expect(capture.mimeType).toBe('image/jpeg'); + expect(stub.drawn).toEqual([{ width: 900, height: 675 }]); + expect(capture.dataBase64.length).toBe(64); + } finally { + stub.restore(); + } + }); + + it('never upscales a small canvas and keeps text-crisp PNG output', async () => { + const stub = stubCanvas(32); + try { + const surface = createSurface(''); + const capture = await captureSurface(surface, 900); + + expect(capture.width).toBe(320); + expect(capture.height).toBe(240); + expect(capture.mimeType).toBe('image/png'); + } finally { + stub.restore(); + } + }); + + it('steps the frame down when one capture exceeds the payload limit', async () => { + const limit = Math.ceil((1_500_000 / 3) * 4); + // The 900px export overflows the limit, so the runtime retries smaller. + const stub = stubCanvas((width) => (width >= 900 ? limit + 16 : 128)); + try { + const surface = createSurface(''); + const capture = await captureSurface(surface, 900); + + expect(stub.drawn.length).toBeGreaterThan(1); + expect(capture.width).toBeLessThan(900); + } finally { + stub.restore(); + } + }); + + it('reports an app that paints no canvas instead of inventing pixels', async () => { + const surface = createSurface('
'); + await expect(captureSurface(surface, 900)).rejects.toMatchObject({ + code: 'CAPABILITY_MISSING', + }); + }); + + it('reports an unreadable frame instead of an empty capture', async () => { + const iframe = Object.defineProperty({}, 'contentDocument', { + get() { + throw new Error('Blocked a frame with origin from accessing a cross-origin frame'); + }, + }) as unknown as HTMLIFrameElement; + await expect(captureSurface({ appId: 'opaque', iframe }, 900)).rejects.toBeInstanceOf( + MiniAppAgentSurfaceError, + ); + }); +}); diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.ts new file mode 100644 index 0000000000..c7cf8ad3ef --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentCapture.ts @@ -0,0 +1,195 @@ +/** + * Pixel observation for an Agent-controlled MiniApp surface. + * + * `inspect` cannot see inside a canvas, so for any canvas-rendered MiniApp — + * a game, a chart, a map, a drawing tool — the pixels *are* the state. This + * module owns that one job: pick the surface region that carries the app's + * output, scale it into the published bounds, and hand back an image. + * + * It depends only on the wire limits from `miniAppAgentProtocol` and on the + * surface's document accessor. It knows nothing about nodes, input, bridging, + * or transport, so a new capture source (video, SVG, a future renderer) is + * added here without touching the action layer. + */ + +import { requireSurfaceDocument, type MiniAppAgentDomSurface } from './miniAppAgentDomRuntime'; +import { + MINIAPP_AGENT_MAX_CAPTURE_BYTES, + MINIAPP_AGENT_MAX_CAPTURE_EDGE, + MINIAPP_AGENT_MAX_NAME_CHARS, + surfaceError, +} from './miniAppAgentProtocol'; + +export interface CaptureResult { + mimeType: string; + dataBase64: string; + width: number; + height: number; + source: string; + label?: string; +} + +/** Largest side at or below this stays PNG: crisp text, no artifacts. */ +const PNG_EDGE_LIMIT = 480; +/** JPEG quality for a scaled frame. */ +const JPEG_QUALITY = 0.85; +/** Same-origin child frames worth looking into for a canvas. */ +const MAX_FRAME_DEPTH = 2; + +interface CanvasCandidate { + canvas: HTMLCanvasElement; + doc: Document; + label: string; +} + +function clip(value: string, limit: number): string | undefined { + const trimmed = value.trim(); + return trimmed ? trimmed.slice(0, limit) : undefined; +} + +function canvasArea(canvas: HTMLCanvasElement): number { + return (canvas.width || 0) * (canvas.height || 0); +} + +function canvasLabel(canvas: HTMLCanvasElement): string { + const id = canvas.getAttribute('id'); + if (id) return `#${clip(id, MINIAPP_AGENT_MAX_NAME_CHARS) ?? ''}`; + const testId = + canvas.getAttribute('data-testid') ?? + canvas.getAttribute('data-test-id') ?? + canvas.getAttribute('data-test'); + if (testId) return clip(testId, MINIAPP_AGENT_MAX_NAME_CHARS) ?? 'canvas'; + return 'canvas'; +} + +/** Every canvas this surface may read, including same-origin child frames. */ +function collectCanvases(doc: Document, depth = 0): CanvasCandidate[] { + const found: CanvasCandidate[] = Array.from(doc.querySelectorAll('canvas')).map((canvas) => ({ + canvas, + doc, + label: canvasLabel(canvas), + })); + if (depth >= MAX_FRAME_DEPTH) return found; + for (const frame of Array.from(doc.querySelectorAll('iframe'))) { + let childDoc: Document | null = null; + try { + childDoc = (frame as HTMLIFrameElement).contentDocument; + } catch { + childDoc = null; + } + if (childDoc) found.push(...collectCanvases(childDoc, depth + 1)); + } + return found; +} + +function nextAnimationFrame(win: Window | null): Promise { + return new Promise((resolve) => { + if (win && typeof win.requestAnimationFrame === 'function') { + win.requestAnimationFrame(() => resolve()); + return; + } + setTimeout(resolve, 16); + }); +} + +function readCanvasPixels(candidate: CanvasCandidate, maxEdge: number): CaptureResult { + const { canvas, doc } = candidate; + const sourceWidth = canvas.width || 0; + const sourceHeight = canvas.height || 0; + if (sourceWidth <= 0 || sourceHeight <= 0) { + throw surfaceError( + 'CAPABILITY_MISSING', + `screenshot: ${candidate.label} has no pixel buffer to capture (width or height is 0).`, + ); + } + + // Shrink in steps so a dense frame still fits the observation limit instead + // of failing outright. Never upscale: a small canvas stays sharp. + const edgesToTry = [maxEdge, Math.round(maxEdge * 0.7), Math.round(maxEdge * 0.5)].filter( + (edge, index, all) => edge >= 320 && all.indexOf(edge) === index, + ); + + let lastError: unknown; + for (const edge of edgesToTry) { + const scale = Math.min(1, edge / Math.max(sourceWidth, sourceHeight)); + const width = Math.max(1, Math.round(sourceWidth * scale)); + const height = Math.max(1, Math.round(sourceHeight * scale)); + const target = doc.createElement('canvas'); + target.width = width; + target.height = height; + const context = target.getContext('2d'); + if (!context) { + throw surfaceError( + 'CAPABILITY_MISSING', + 'screenshot: this surface has no 2D drawing context, so canvas pixels cannot be captured.', + ); + } + try { + context.drawImage(canvas, 0, 0, width, height); + } catch (error) { + // Tainted canvas: cross-origin media the app painted cannot be read back. + throw surfaceError( + 'CAPABILITY_MISSING', + `screenshot: ${candidate.label} paints pixels the host may not read: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const mimeType = Math.max(width, height) <= PNG_EDGE_LIMIT ? 'image/png' : 'image/jpeg'; + let dataUrl: string; + try { + dataUrl = + mimeType === 'image/png' + ? target.toDataURL('image/png') + : target.toDataURL('image/jpeg', JPEG_QUALITY); + } catch (error) { + throw surfaceError( + 'CAPABILITY_MISSING', + `screenshot: ${candidate.label} cannot be exported as an image: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + } + const separator = dataUrl.indexOf(','); + if (separator < 0) { + lastError = surfaceError('HOST_ERROR', 'screenshot: the canvas export produced no data.'); + continue; + } + const dataBase64 = dataUrl.slice(separator + 1); + if (dataBase64.length > (MINIAPP_AGENT_MAX_CAPTURE_BYTES / 3) * 4) { + lastError = surfaceError( + 'HOST_ERROR', + 'screenshot: the captured frame is larger than the observation limit.', + ); + continue; + } + return { mimeType, dataBase64, width, height, source: 'canvas', label: candidate.label }; + } + throw lastError instanceof Error + ? lastError + : surfaceError('HOST_ERROR', 'screenshot: the frame could not be captured.'); +} + +/** + * Capture what the MiniApp is painting right now. + * + * The read happens inside an animation frame: a WebGL canvas may present an + * empty drawing buffer outside the frame that draws it. + */ +export async function captureSurface( + surface: MiniAppAgentDomSurface, + maxEdge: number, +): Promise { + const doc = requireSurfaceDocument(surface); + const candidates = collectCanvases(doc) + .filter((candidate) => canvasArea(candidate.canvas) > 0) + .sort((left, right) => canvasArea(right.canvas) - canvasArea(left.canvas)); + if (candidates.length === 0) { + throw surfaceError( + 'CAPABILITY_MISSING', + 'screenshot: this MiniApp paints no canvas, so there are no pixels to capture. Read its DOM instead, or use the app bridge for internal state.', + ); + } + await nextAnimationFrame(doc.defaultView); + return readCanvasPixels(candidates[0], Math.min(maxEdge, MINIAPP_AGENT_MAX_CAPTURE_EDGE)); +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.test.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.test.ts new file mode 100644 index 0000000000..2cce7f6e32 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.test.ts @@ -0,0 +1,227 @@ +/** + * @vitest-environment jsdom + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + canReadSurfaceDocument, + clickNode, + findNodes, + inspectDocument, + pressKey, + readNode, + scrollSurface, + typeIntoNode, + type MiniAppAgentDomSurface, +} from './miniAppAgentDomRuntime'; +import { MINIAPP_AGENT_MAX_NODES, MiniAppAgentSurfaceError } from './miniAppAgentProtocol'; + +const TOOLBAR_HTML = ` +
+

Pomodoro

+ + + +
+ +
+`; + +function createSurface(html = TOOLBAR_HTML): MiniAppAgentDomSurface { + const doc = document.implementation.createHTMLDocument('Pomodoro'); + doc.body.innerHTML = html; + const iframe = { contentDocument: doc } as unknown as HTMLIFrameElement; + return { appId: 'demo', iframe }; +} + +function unreadableSurface(): MiniAppAgentDomSurface { + const getter = () => { + throw new Error('Blocked a frame with origin from accessing a cross-origin frame'); + }; + const iframe = Object.defineProperty({}, 'contentDocument', { get: getter }) as unknown as HTMLIFrameElement; + return { appId: 'opaque', iframe }; +} + +function errorCode(run: () => unknown): string { + try { + run(); + } catch (error) { + expect(error).toBeInstanceOf(MiniAppAgentSurfaceError); + return (error as MiniAppAgentSurfaceError).code; + } + throw new Error('expected a typed surface error'); +} + +function flatNodes(nodes: ReturnType['tree']): { role?: string; name?: string }[] { + return nodes.flatMap((node) => [node, ...flatNodes(node.children ?? [])]); +} + +describe('MiniApp agent DOM runtime', () => { + it('keeps old handles stale after the entire Web UI runtime reloads', async () => { + vi.resetModules(); + const before = await import('./miniAppAgentDomRuntime'); + const first = createSurface(); + const oldKey = before.findNodes(first, { testId: 'start-button' })[0].nodeKey!; + vi.resetModules(); + const after = await import('./miniAppAgentDomRuntime'); + const second = createSurface(); + const newKey = after.findNodes(second, { testId: 'start-button' })[0].nodeKey!; + const click = vi.fn(); + second.iframe.contentDocument!.querySelector('button')!.addEventListener('click', click); + expect(newKey).not.toBe(oldKey); + expect(() => after.clickNode(second, oldKey)).toThrow('STALE_TARGET'); + expect(click).not.toHaveBeenCalled(); + }); + + afterEach(() => document.body.replaceChildren()); + + it('uses the native setter for each editable field type', () => { + const iframe = document.createElement('iframe'); + document.body.append(iframe); + iframe.contentDocument!.body.innerHTML = ''; + const surface = { appId: 'fields', iframe }; + for (const tag of ['input', 'textarea', 'select']) { + const node = findNodes(surface, { selector: tag })[0]; + const text = tag === 'textarea' ? ' first\nsecond ' : 'b'; + typeIntoNode(surface, node.nodeKey!, text); + expect((iframe.contentDocument!.querySelector(tag) as HTMLInputElement).value).toBe(text); + } + }); + + it('never reuses handles after the frame or its document is replaced', () => { + const original = createSurface(''); + const key = findNodes(original, { role: 'button' })[0].nodeKey!; + const replacement = createSurface(''); + const nextKey = findNodes(replacement, { role: 'button' })[0].nodeKey!; + expect(nextKey).not.toBe(key); + expect(errorCode(() => clickNode(replacement, key))).toBe('STALE_TARGET'); + + // The old Document still has connected nodes. Connectivity alone must not + // allow actions against the document that the frame no longer presents. + Object.defineProperty(original.iframe, 'contentDocument', { value: replacement.iframe.contentDocument }); + expect(errorCode(() => clickNode(original, key))).toBe('STALE_TARGET'); + }); + + it('inspects a bounded, accessibility-shaped tree with node handles', () => { + const surface = createSurface(); + const result = inspectDocument(surface); + const nodes = flatNodes(result.tree); + + expect(result.title).toBe('Pomodoro'); + expect(result.truncated).toBe(false); + expect(nodes.some((node) => node.role === 'button' && node.name === 'Start timer')).toBe(true); + expect(nodes.some((node) => node.role === 'heading' && node.name === 'Pomodoro')).toBe(true); + // Hidden containers, scripts, and bare structural wrappers stay out. + expect(nodes.some((node) => node.name === 'Hidden')).toBe(false); + expect(JSON.stringify(result.tree)).not.toContain('node-0'); + }); + + it('reuses node handles across observations and finds by priority axes', () => { + const surface = createSurface(); + const first = inspectDocument(surface); + const second = inspectDocument(surface); + expect(JSON.stringify(second.tree)).toBe(JSON.stringify(first.tree)); + + const byTestId = findNodes(surface, { testId: 'start-button' }); + expect(byTestId).toHaveLength(1); + expect(byTestId[0].name).toBe('Start timer'); + + const byRole = findNodes(surface, { role: 'button', name: 'Start timer', exact: true }); + expect(byRole).toHaveLength(1); + expect(byRole[0].testId).toBe('start-button'); + + const byText = findNodes(surface, { text: 'Stop' }); + expect(byText).toHaveLength(1); + + const bySelector = findNodes(surface, { selector: '#label' }); + expect(bySelector).toHaveLength(1); + expect(bySelector[0].role).toBe('textbox'); + }); + + it('reports a typed error for an invalid selector instead of throwing raw', () => { + const surface = createSurface(); + expect(errorCode(() => findNodes(surface, { selector: 'button[' }))).toBe('INVALID_PARAMS'); + }); + + it('reads, types, clicks, and presses keys through real DOM events', async () => { + const surface = createSurface(); + const seen: string[] = []; + const doc = surface.iframe.contentDocument as Document; + doc.addEventListener('click', () => seen.push('click')); + doc.addEventListener('input', () => seen.push('input')); + + const start = findNodes(surface, { testId: 'start-button' })[0]; + const field = findNodes(surface, { selector: '#label' })[0]; + + expect(clickNode(surface, start.nodeKey!).detail.role).toBe('button'); + const typed = typeIntoNode(surface, field.nodeKey!, 'deep work', true); + expect(typed.detail.value).toBe('deep work'); + const read = readNode(surface, field.nodeKey!); + expect(read.value).toBe('deep work'); + expect(read.attributes['aria-label']).toBe('Session label'); + + await pressKey(surface, 'Enter'); + expect(seen).toEqual(expect.arrayContaining(['click', 'input'])); + }); + + it('maps game keys to DOM code values and holds them before release', async () => { + const surface = createSurface(''); + const doc = surface.iframe.contentDocument as Document; + const events: { type: string; key: string; code: string; at: number }[] = []; + const started = Date.now(); + for (const type of ['keydown', 'keyup']) { + doc.addEventListener(type, (event) => { + const keyboard = event as KeyboardEvent; + events.push({ type, key: keyboard.key, code: keyboard.code, at: Date.now() - started }); + }); + } + + await pressKey(surface, ' ', { holdMs: 40 }); + await pressKey(surface, 'Left'); + + expect(events.map((event) => event.type)).toEqual(['keydown', 'keyup', 'keydown', 'keyup']); + // A game reads `code` on its own frame; the DOM name alone is not enough. + expect(events[0]).toMatchObject({ key: ' ', code: 'Space' }); + expect(events[2]).toMatchObject({ key: 'ArrowLeft', code: 'ArrowLeft' }); + // The key has to stay down long enough for a held-input game loop to see it. + expect(events[1].at - events[0].at).toBeGreaterThanOrEqual(30); + expect(events[3].at - events[2].at).toBeLessThan(30); + }); + + it('refuses disabled targets and unknown handles with typed errors', () => { + const surface = createSurface(); + const stop = findNodes(surface, { selector: '#stop' })[0]; + expect(errorCode(() => clickNode(surface, stop.nodeKey!))).toBe('INVALID_PARAMS'); + const unknown = stop.nodeKey!.replace(/\d+$/, '999999'); + expect(errorCode(() => clickNode(surface, unknown))).toBe('UNKNOWN_TARGET'); + }); + + it('invalidates handles once the document is replaced', () => { + const surface = createSurface(); + const start = findNodes(surface, { testId: 'start-button' })[0]; + (surface.iframe.contentDocument as Document).body.innerHTML = ''; + expect(errorCode(() => clickNode(surface, start.nodeKey!))).toBe('STALE_TARGET'); + }); + + it('scrolls the document and reports the applied offset', () => { + const surface = createSurface(); + const result = scrollSurface(surface, undefined, 0, 240) as { scrollTop: number; deltaY: number }; + expect(result.deltaY).toBe(240); + expect(result.scrollTop).toBeGreaterThanOrEqual(0); + }); + + it('truncates the observation at the contract budget', () => { + const rows = Array.from({ length: MINIAPP_AGENT_MAX_NODES + 40 }, (_, index) => ``).join(''); + const surface = createSurface(`
${rows}
`); + const result = inspectDocument(surface); + expect(result.truncated).toBe(true); + expect(flatNodes(result.tree).length).toBeLessThanOrEqual(MINIAPP_AGENT_MAX_NODES); + }); + + it('reports an unreadable frame as a surface capability gap', () => { + const surface = unreadableSurface(); + expect(canReadSurfaceDocument(surface)).toBe(false); + expect(errorCode(() => inspectDocument(surface))).toBe('SURFACE_UNAVAILABLE'); + }); +}); diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.ts new file mode 100644 index 0000000000..6e580f05b1 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentDomRuntime.ts @@ -0,0 +1,744 @@ +/** + * DOM runtime for the Agent-controlled MiniApp surface. + * + * The MiniApp lives in an iframe owned by this window, so only the Web UI can + * read its document and dispatch input. Everything here is framework-free and + * takes an explicit surface, which keeps the transport (bridge/router) and the + * React lifecycle (registry/hook) independent of DOM details. + * + * Two invariants: + * - a node handle is only valid while its element is still connected, so a + * reload turns into `STALE_TARGET` instead of acting on a lookalike node; + * - every failure is a typed error, never a silent no-op. + */ + +import { + MINIAPP_AGENT_MAX_DEPTH, + MINIAPP_AGENT_MAX_KEY_HOLD_MS, + MINIAPP_AGENT_MAX_MATCHES, + MINIAPP_AGENT_MAX_NAME_CHARS, + MINIAPP_AGENT_MAX_NODES, + MINIAPP_AGENT_MAX_TEXT_CHARS, + surfaceError, + createMiniAppDocumentId, + type MiniAppAgentLocator, + type MiniAppAgentNode, +} from './miniAppAgentProtocol'; + +export interface MiniAppAgentDomSurface { + appId: string; + iframe: HTMLIFrameElement; +} + +const IGNORED_TAGS = new Set([ + 'script', + 'style', + 'noscript', + 'template', + 'meta', + 'link', + 'head', + 'title', + 'base', +]); + +const INTERACTIVE_TAGS = new Set([ + 'a', + 'area', + 'button', + 'details', + 'dialog', + 'embed', + 'iframe', + 'input', + 'label', + 'option', + 'output', + 'progress', + 'select', + 'summary', + 'textarea', +]); + +const STRUCTURAL_ROLES: Record = { + a: 'link', + article: 'article', + aside: 'complementary', + body: 'document', + button: 'button', + dialog: 'dialog', + form: 'form', + h1: 'heading', + h2: 'heading', + h3: 'heading', + h4: 'heading', + h5: 'heading', + h6: 'heading', + header: 'banner', + img: 'img', + input: 'textbox', + li: 'listitem', + main: 'main', + nav: 'navigation', + ol: 'list', + option: 'option', + p: 'paragraph', + select: 'combobox', + table: 'table', + td: 'cell', + textarea: 'textbox', + th: 'columnheader', + tr: 'row', + ul: 'list', +}; + +const INPUT_ROLES: Record = { + button: 'button', + checkbox: 'checkbox', + color: 'button', + date: 'textbox', + email: 'textbox', + file: 'button', + image: 'button', + number: 'spinbutton', + password: 'textbox', + radio: 'radio', + range: 'slider', + reset: 'button', + search: 'searchbox', + submit: 'button', + tel: 'textbox', + text: 'textbox', + time: 'textbox', + url: 'textbox', +}; + +const TEST_ID_ATTRIBUTES = ['data-testid', 'data-test-id', 'data-test', 'data-openbitfun-testid']; +const READ_ATTRIBUTES = ['id', 'name', 'type', 'placeholder', 'aria-label', 'title', 'href', 'role']; + +/** Node handles never alias another frame or document, even after a reload. */ +class SurfaceNodeIndex { + private readonly keys = new WeakMap(); + private readonly elements = new Map(); + private next = 1; + private readonly prefix = `node-${createMiniAppDocumentId()}-`; + + constructor(readonly document: Document) {} + + keyOf(element: Element): string { + const existing = this.keys.get(element); + if (existing) return existing; + const key = `${this.prefix}${this.next}`; + this.next += 1; + this.keys.set(element, key); + this.elements.set(key, element); + return key; + } + + resolve(nodeKey: string): Element { + if (!nodeKey.startsWith(this.prefix)) { + throw surfaceError('STALE_TARGET', 'The node belongs to another MiniApp document; run inspect again.'); + } + const element = this.elements.get(nodeKey); + if (!element) { + throw surfaceError('UNKNOWN_TARGET', 'The node handle is unknown; run inspect again.'); + } + if (!element.isConnected || element.ownerDocument !== this.document) { + throw surfaceError( + 'STALE_TARGET', + 'The MiniApp document changed after this node handle was observed; run inspect again.', + ); + } + return element; + } +} + +const nodeIndexes = new WeakMap(); + +function nodeIndex(surface: MiniAppAgentDomSurface): SurfaceNodeIndex { + const doc = requireSurfaceDocument(surface); + const existing = nodeIndexes.get(surface.iframe); + if (existing?.document === doc) return existing; + const created = new SurfaceNodeIndex(doc); + nodeIndexes.set(surface.iframe, created); + return created; +} + +/** Whether this surface exposes a readable document at all. */ +export function canReadSurfaceDocument(surface: MiniAppAgentDomSurface): boolean { + try { + const doc = surface.iframe.contentDocument; + return Boolean(doc?.documentElement); + } catch { + return false; + } +} + +/** + * The surface document, or a typed `SURFACE_UNAVAILABLE`. + * + * Exported because any read of a MiniApp surface starts here, including pixel + * capture, which lives in its own module precisely so it does not need the rest + * of this one. + */ +export function requireSurfaceDocument(surface: MiniAppAgentDomSurface): Document { + let doc: Document | null = null; + try { + doc = surface.iframe.contentDocument; + } catch { + doc = null; + } + if (!doc?.documentElement) { + throw surfaceError( + 'SURFACE_UNAVAILABLE', + 'The MiniApp frame document is not readable from this surface: it runs with an opaque origin or is not mounted yet.', + ); + } + return doc; +} + +function boundText(value: string | null | undefined, limit: number): string | undefined { + const normalized = (value ?? '').replace(/\s+/g, ' ').trim(); + if (!normalized) return undefined; + return normalized.length > limit ? `${normalized.slice(0, limit)}…` : normalized; +} + +function isElementVisible(element: Element): boolean { + if (element.hasAttribute('hidden')) return false; + if (element.getAttribute('aria-hidden') === 'true') return false; + const style = (element as HTMLElement).style; + if (style?.display === 'none' || style?.visibility === 'hidden') return false; + if (element.tagName.toLowerCase() === 'input' && element.getAttribute('type') === 'hidden') { + return false; + } + return true; +} + +function implicitRole(element: Element): string | undefined { + const tag = element.tagName.toLowerCase(); + if (tag === 'input') { + const type = (element.getAttribute('type') ?? 'text').toLowerCase(); + return INPUT_ROLES[type] ?? 'textbox'; + } + if (tag === 'a') return element.hasAttribute('href') ? 'link' : undefined; + return STRUCTURAL_ROLES[tag]; +} + +function nodeRole(element: Element): string { + const explicit = element.getAttribute('role')?.trim(); + if (explicit) return explicit.split(/\s+/)[0].toLowerCase(); + return implicitRole(element) ?? element.tagName.toLowerCase(); +} + +function nodeTestId(element: Element): string | undefined { + for (const attribute of TEST_ID_ATTRIBUTES) { + const value = element.getAttribute(attribute)?.trim(); + if (value) return value; + } + return undefined; +} + +function labelledByText(element: Element): string | undefined { + const ids = element.getAttribute('aria-labelledby')?.trim(); + if (!ids) return undefined; + const doc = element.ownerDocument; + const text = ids + .split(/\s+/) + .map((id) => doc.getElementById(id)?.textContent ?? '') + .join(' ') + .trim(); + return text || undefined; +} + +function labelsText(element: Element): string | undefined { + const labels = (element as HTMLInputElement).labels; + if (!labels || typeof labels.length !== 'number') return undefined; + const text = Array.from(labels as unknown as Element[]) + .map((label) => label.textContent ?? '') + .join(' ') + .trim(); + return text || undefined; +} + +/** Accessible name, in the order a user would read it. */ +function nodeName(element: Element): string | undefined { + return ( + boundText(element.getAttribute('aria-label'), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(labelledByText(element), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(labelsText(element), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(element.getAttribute('alt'), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(element.getAttribute('title'), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(element.getAttribute('placeholder'), MINIAPP_AGENT_MAX_NAME_CHARS) + ?? boundText(element.textContent, MINIAPP_AGENT_MAX_NAME_CHARS) + ); +} + +function nodeValue(element: Element): string | undefined { + const tag = element.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea' || tag === 'select') { + return boundText((element as HTMLInputElement).value, MINIAPP_AGENT_MAX_TEXT_CHARS); + } + if ((element as HTMLElement).isContentEditable) { + return boundText(element.textContent, MINIAPP_AGENT_MAX_TEXT_CHARS); + } + return undefined; +} + +function nodeChecked(element: Element): boolean | undefined { + const tag = element.tagName.toLowerCase(); + if (tag !== 'input') return undefined; + const type = (element.getAttribute('type') ?? '').toLowerCase(); + if (type !== 'checkbox' && type !== 'radio') return undefined; + return (element as HTMLInputElement).checked; +} + +function nodeEnabled(element: Element): boolean | undefined { + if (element.getAttribute('aria-disabled') === 'true') return false; + const disabled = (element as HTMLButtonElement).disabled; + if (typeof disabled === 'boolean' && (INTERACTIVE_TAGS.has(element.tagName.toLowerCase()) + || element.hasAttribute('contenteditable'))) { + return !disabled; + } + return undefined; +} + +/** Whether a node carries information worth reporting to the model. */ +function isDescribable(element: Element): boolean { + const tag = element.tagName.toLowerCase(); + if (INTERACTIVE_TAGS.has(tag)) return true; + if (element.hasAttribute('role') || element.hasAttribute('aria-label')) return true; + if (nodeTestId(element)) return true; + if (element.hasAttribute('contenteditable')) return true; + if (tag === 'h1' || tag === 'h2' || tag === 'h3' || tag === 'h4' || tag === 'h5' || tag === 'h6') return true; + if (tag === 'img') return Boolean(element.getAttribute('alt')); + const text = boundText(element.textContent, MINIAPP_AGENT_MAX_TEXT_CHARS); + return Boolean(text) && element.children.length === 0; +} + +function describeNode(element: Element): MiniAppAgentNode { + const text = boundText(element.textContent, MINIAPP_AGENT_MAX_TEXT_CHARS); + const value = nodeValue(element); + return { + role: nodeRole(element), + name: nodeName(element), + tag: element.tagName.toLowerCase().slice(0, 32), + testId: nodeTestId(element), + value, + text, + enabled: nodeEnabled(element), + checked: nodeChecked(element), + children: [], + }; +} + +export interface InspectOptions { + maxNodes?: number; + maxDepth?: number; +} + +export interface InspectResult { + url?: string; + title?: string; + nodeCount: number; + truncated: boolean; + tree: MiniAppAgentNode[]; +} + +/** Bounded, a11y-shaped observation of the MiniApp document. */ +export function inspectDocument( + surface: MiniAppAgentDomSurface, + options: InspectOptions = {}, +): InspectResult { + const doc = requireSurfaceDocument(surface); + const index = nodeIndex(surface); + const maxNodes = Math.max(1, options.maxNodes ?? MINIAPP_AGENT_MAX_NODES); + const maxDepth = Math.max(1, options.maxDepth ?? MINIAPP_AGENT_MAX_DEPTH); + const budget = { remaining: maxNodes, truncated: false }; + const root = doc.body ?? doc.documentElement; + const tree: MiniAppAgentNode[] = []; + + const walk = (parent: Element, depth: number, out: MiniAppAgentNode[]) => { + if (depth > maxDepth) { + budget.truncated = true; + return; + } + for (const child of Array.from(parent.children)) { + const tag = child.tagName.toLowerCase(); + if (IGNORED_TAGS.has(tag) || !isElementVisible(child)) continue; + if (!isDescribable(child)) { + walk(child, depth, out); + continue; + } + if (budget.remaining <= 0) { + budget.truncated = true; + return; + } + budget.remaining -= 1; + const node = describeNode(child); + node.nodeKey = index.keyOf(child); + const children: MiniAppAgentNode[] = []; + walk(child, depth + 1, children); + if (children.length > 0) node.children = children; + out.push(node); + if (budget.truncated) return; + } + }; + + walk(root, 1, tree); + const href = doc.defaultView?.location?.href; + return { + url: href && href !== 'about:blank' ? href : undefined, + title: boundText(doc.title, MINIAPP_AGENT_MAX_NAME_CHARS), + nodeCount: maxNodes - budget.remaining, + truncated: budget.truncated, + tree, + }; +} + +function matchesLocator(element: Element, locator: MiniAppAgentLocator): boolean { + const exact = locator.exact === true; + if (locator.testId && nodeTestId(element) !== locator.testId) return false; + if (locator.role && nodeRole(element) !== locator.role.toLowerCase()) return false; + if (locator.name) { + const name = nodeName(element); + if (!name) return false; + if (exact ? name !== locator.name : !name.includes(locator.name)) return false; + } + if (locator.text) { + const text = boundText(element.textContent, MINIAPP_AGENT_MAX_TEXT_CHARS); + if (!text) return false; + if (exact ? text !== locator.text : !text.includes(locator.text)) return false; + } + return true; +} + +/** Flat list of matching nodes, highest-priority axis first. */ +export function findNodes( + surface: MiniAppAgentDomSurface, + locator: MiniAppAgentLocator, +): MiniAppAgentNode[] { + const doc = requireSurfaceDocument(surface); + const index = nodeIndex(surface); + let candidates: Element[]; + if (locator.selector) { + try { + candidates = Array.from(doc.querySelectorAll(locator.selector)); + } catch { + throw surfaceError('INVALID_PARAMS', `The CSS selector is not valid in this document: ${locator.selector}`); + } + } else { + const body = doc.body ?? doc.documentElement; + candidates = Array.from(body.querySelectorAll('*')); + } + + const semanticMatch = Boolean(locator.text || locator.name); + const matched: Element[] = []; + for (const element of candidates) { + if (IGNORED_TAGS.has(element.tagName.toLowerCase())) continue; + if (!isElementVisible(element)) continue; + if (!matchesLocator(element, locator)) continue; + matched.push(element); + } + + // A name/text match also hits every ancestor that contains the label, so keep + // the most specific node and drop the containers around it. + const specific = semanticMatch + ? matched.filter((element) => !matched.some( + (other) => other !== element && element.contains(other), + )) + : matched; + + const matches: MiniAppAgentNode[] = []; + for (const element of specific) { + const node = describeNode(element); + node.nodeKey = index.keyOf(element); + matches.push(node); + if (matches.length >= MINIAPP_AGENT_MAX_MATCHES) break; + } + return matches; +} + +function resolveElement(surface: MiniAppAgentDomSurface, nodeKey: string): Element { + requireSurfaceDocument(surface); + return nodeIndex(surface).resolve(nodeKey); +} + +export interface ReadResult { + text?: string; + value?: string; + attributes: Record; +} + +export function readNode( + surface: MiniAppAgentDomSurface, + nodeKey: string, + attribute?: string, +): ReadResult { + const element = resolveElement(surface, nodeKey); + const attributes: Record = {}; + if (attribute) { + const value = element.getAttribute(attribute); + if (value !== null) attributes[attribute] = boundText(value, MINIAPP_AGENT_MAX_TEXT_CHARS) ?? ''; + } else { + for (const name of READ_ATTRIBUTES) { + const value = element.getAttribute(name); + if (value) attributes[name] = boundText(value, MINIAPP_AGENT_MAX_TEXT_CHARS) ?? ''; + } + } + return { + text: boundText(element.textContent, MINIAPP_AGENT_MAX_TEXT_CHARS), + value: nodeValue(element), + attributes, + }; +} + +function assertActionable(element: Element, action: string): void { + if (element.getAttribute('aria-disabled') === 'true' || (element as HTMLButtonElement).disabled === true) { + throw surfaceError( + 'INVALID_PARAMS', + `${action}: the target is disabled and cannot receive input.`, + ); + } +} + +function scrollIntoView(element: Element): void { + const scrollIntoViewFn = (element as HTMLElement).scrollIntoView; + if (typeof scrollIntoViewFn === 'function') { + try { + scrollIntoViewFn.call(element, { block: 'center', inline: 'center' }); + } catch { + // Scrolling is best-effort; dispatch still proceeds. + } + } +} + +function createEvent( + doc: Document, + name: string, + init: Record, + ctor: 'MouseEvent' | 'PointerEvent' | 'KeyboardEvent' | 'Event', +): T { + const view = doc.defaultView as (Window & typeof globalThis) | null; + const EventCtor = (view?.[ctor] ?? (globalThis as unknown as Record)[ctor]) as + | (new (type: string, init: Record) => T) + | undefined; + if (EventCtor) return new EventCtor(name, init); + return new Event(name, init as EventInit) as T; +} + +export interface InputResult { + target: string; + detail: Record; +} + +export function clickNode(surface: MiniAppAgentDomSurface, nodeKey: string): InputResult { + const element = resolveElement(surface, nodeKey); + assertActionable(element, 'click'); + const doc = element.ownerDocument; + scrollIntoView(element); + const focusable = element as HTMLElement; + if (typeof focusable.focus === 'function') { + try { + focusable.focus({ preventScroll: true }); + } catch { + // Focus is best-effort for non-focusable nodes. + } + } + const init = { bubbles: true, cancelable: true, composed: true, view: doc.defaultView, button: 0 }; + element.dispatchEvent(createEvent(doc, 'pointerdown', { ...init, isPrimary: true }, 'PointerEvent')); + element.dispatchEvent(createEvent(doc, 'mousedown', init, 'MouseEvent')); + element.dispatchEvent(createEvent(doc, 'pointerup', { ...init, isPrimary: true }, 'PointerEvent')); + element.dispatchEvent(createEvent(doc, 'mouseup', init, 'MouseEvent')); + element.dispatchEvent(createEvent(doc, 'click', init, 'MouseEvent')); + return { + target: nodeKey, + detail: { role: nodeRole(element), name: nodeName(element) ?? null }, + }; +} + +function setInputValue(element: Element, text: string, clear: boolean): void { + const field = element as HTMLInputElement; + const previous = typeof field.value === 'string' ? field.value : ''; + const next = clear ? text : `${previous}${text}`; + const view = element.ownerDocument.defaultView as (Window & typeof globalThis) | null; + const tag = element.tagName.toLowerCase(); + const constructor = tag === 'textarea' ? view?.HTMLTextAreaElement + : tag === 'select' ? view?.HTMLSelectElement : view?.HTMLInputElement; + const setter = constructor + ? Object.getOwnPropertyDescriptor(constructor.prototype, 'value')?.set + : undefined; + if (setter) { + setter.call(element, next); + } else { + field.value = next; + } +} + +export function typeIntoNode( + surface: MiniAppAgentDomSurface, + nodeKey: string, + text: string, + clear = true, +): InputResult { + const element = resolveElement(surface, nodeKey); + assertActionable(element, 'type'); + const doc = element.ownerDocument; + scrollIntoView(element); + const editable = element as HTMLElement; + if (typeof editable.focus === 'function') { + try { + editable.focus({ preventScroll: true }); + } catch { + // Typing into a non-focusable node still updates its value below. + } + } + + const tag = element.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea' || tag === 'select') { + setInputValue(element, text, clear); + } else if (editable.isContentEditable) { + editable.textContent = clear ? text : `${editable.textContent ?? ''}${text}`; + } else { + throw surfaceError( + 'INVALID_PARAMS', + 'type: the target is not an editable field. Use a textbox, textarea, select, or contenteditable node.', + ); + } + + element.dispatchEvent(createEvent(doc, 'input', { bubbles: true, composed: true, inputType: 'insertText', data: text }, 'Event')); + element.dispatchEvent(createEvent(doc, 'change', { bubbles: true, composed: true }, 'Event')); + return { target: nodeKey, detail: { value: nodeValue(element) ?? null } }; +} + +const MODIFIER_KEYS = new Set(['alt', 'control', 'meta', 'shift']); + +/** + * `KeyboardEvent.code` for keys whose DOM name is not a single character. A game + * almost always keys off `code`, so sending `Key ` for Space or leaving arrows + * unmapped makes the app ignore input entirely. + */ +const KEY_CODES: Record = { + ' ': 'Space', + space: 'Space', + spacebar: 'Space', + enter: 'Enter', + return: 'Enter', + escape: 'Escape', + esc: 'Escape', + tab: 'Tab', + backspace: 'Backspace', + delete: 'Delete', + arrowleft: 'ArrowLeft', + left: 'ArrowLeft', + arrowright: 'ArrowRight', + right: 'ArrowRight', + arrowup: 'ArrowUp', + up: 'ArrowUp', + arrowdown: 'ArrowDown', + down: 'ArrowDown', + shift: 'ShiftLeft', + control: 'ControlLeft', + alt: 'AltLeft', + meta: 'MetaLeft', +}; + +function keyCodeFor(key: string): string { + const known = KEY_CODES[key.toLowerCase()]; + if (known) return known; + if (key.length === 1) { + if (/[a-z]/i.test(key)) return `Key${key.toUpperCase()}`; + if (/[0-9]/.test(key)) return `Digit${key}`; + return key; + } + return key; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +} + +/** DOM `key` values that differ from the name a caller is likely to pass. */ +const KEY_VALUE_ALIASES: Record = { + space: ' ', + spacebar: ' ', + esc: 'Escape', + return: 'Enter', + left: 'ArrowLeft', + right: 'ArrowRight', + up: 'ArrowUp', + down: 'ArrowDown', +}; + +export interface KeyOptions { + modifiers?: string[]; + /** Keep the key down this long before releasing it (ms). */ + holdMs?: number; +} + +export async function pressKey( + surface: MiniAppAgentDomSurface, + key: string, + options: KeyOptions = {}, +): Promise { + const doc = requireSurfaceDocument(surface); + // A space is a real key for a game, so only trim the names that carry padding. + const normalizedKey = key === ' ' ? ' ' : key.trim(); + if (!normalizedKey) { + throw surfaceError('INVALID_PARAMS', 'key: the key name is required.'); + } + const target = (doc.activeElement ?? doc.body ?? doc.documentElement) as Element | null; + if (!target) { + throw surfaceError('SURFACE_UNAVAILABLE', 'key: the MiniApp document has no focus target.'); + } + const resolved = (options.modifiers ?? []) + .map((modifier) => modifier.trim().toLowerCase()) + .filter((modifier) => MODIFIER_KEYS.has(modifier)); + const init = { + key: KEY_VALUE_ALIASES[normalizedKey.toLowerCase()] ?? normalizedKey, + code: keyCodeFor(normalizedKey), + bubbles: true, + cancelable: true, + composed: true, + altKey: resolved.includes('alt'), + ctrlKey: resolved.includes('control'), + metaKey: resolved.includes('meta'), + shiftKey: resolved.includes('shift'), + }; + target.dispatchEvent(createEvent(doc, 'keydown', init, 'KeyboardEvent')); + if (!MODIFIER_KEYS.has(normalizedKey.toLowerCase())) { + target.dispatchEvent(createEvent(doc, 'keypress', init, 'KeyboardEvent')); + } + // A game loop samples held keys on its own animation frame, so releasing in + // the same tick reads as "not pressed". Hold first, then release. + const holdMs = Math.min( + Math.max(0, Math.round(options.holdMs ?? 0)), + MINIAPP_AGENT_MAX_KEY_HOLD_MS, + ); + if (holdMs > 0) await delay(holdMs); + target.dispatchEvent(createEvent(doc, 'keyup', init, 'KeyboardEvent')); + return { target: target.tagName?.toLowerCase() ?? 'document', detail: { key: normalizedKey, holdMs, modifiers: resolved } }; +} + +export function scrollSurface( + surface: MiniAppAgentDomSurface, + nodeKey: string | undefined, + deltaX: number, + deltaY: number, +): Record { + const doc = requireSurfaceDocument(surface); + const target = nodeKey + ? resolveElement(surface, nodeKey) + : (doc.scrollingElement ?? doc.documentElement); + const scroller = target as HTMLElement; + scroller.scrollLeft = (scroller.scrollLeft ?? 0) + deltaX; + scroller.scrollTop = (scroller.scrollTop ?? 0) + deltaY; + target.dispatchEvent(createEvent(doc, 'scroll', { bubbles: false }, 'Event')); + return { + scrollTop: scroller.scrollTop ?? 0, + scrollLeft: scroller.scrollLeft ?? 0, + deltaX, + deltaY, + }; +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentProtocol.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentProtocol.ts new file mode 100644 index 0000000000..1591484447 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentProtocol.ts @@ -0,0 +1,169 @@ +/** + * Wire contract for driving a MiniApp surface from the Agent. + * + * Mirrors `openbitfun_product_domains::miniapp::web_surface`. The Rust side owns + * the authoritative limits and error taxonomy; this module only re-declares the + * subset the Web UI must honor, so the surface runtime cannot drift silently. + * + * Errors cross the transport as `": "` strings; the host port + * classifies them by that prefix instead of parsing prose. + */ + +export const MINIAPP_AGENT_SCHEMA_VERSION = 'agent.v1'; + +export const MINIAPP_AGENT_MAX_NODES = 320; +export const MINIAPP_AGENT_MAX_DEPTH = 12; +export const MINIAPP_AGENT_MAX_NAME_CHARS = 120; +export const MINIAPP_AGENT_MAX_TEXT_CHARS = 400; +export const MINIAPP_AGENT_MAX_BRIDGE_BYTES = 32 * 1024; +export const MINIAPP_AGENT_MAX_BRIDGE_CAPABILITIES = 8; +/** Declared action names are advisory: bounded so a verbose app cannot flood the turn. */ +export const MINIAPP_AGENT_MAX_BRIDGE_ACTIONS = 32; +export const MINIAPP_AGENT_MAX_ACTION_NAME_CHARS = 48; +export const MINIAPP_AGENT_BRIDGE_TIMEOUT_MS = 5_000; +export const MINIAPP_AGENT_MAX_MATCHES = 20; +export const MINIAPP_AGENT_MAX_CAPTURE_EDGE = 1024; +export const MINIAPP_AGENT_DEFAULT_CAPTURE_EDGE = 900; +export const MINIAPP_AGENT_MAX_CAPTURE_BYTES = 1_500_000; +export const MINIAPP_AGENT_MAX_KEY_HOLD_MS = 2_000; +/** Sampling window bounds for `watch`: one call replaces a per-turn observe loop. */ +export const MINIAPP_AGENT_MAX_WATCH_MS = 120_000; +export const MINIAPP_AGENT_MIN_WATCH_POLL_MS = 250; +export const MINIAPP_AGENT_DEFAULT_WATCH_MS = 20_000; +export const MINIAPP_AGENT_DEFAULT_WATCH_POLL_MS = 1_000; +export const MINIAPP_AGENT_MAX_WATCH_SAMPLES = 120; +/** Total serialized budget for one watch timeline; sampling stops before it overflows. */ +export const MINIAPP_AGENT_MAX_WATCH_SAMPLE_BUDGET = 24 * 1024; + +export const MINIAPP_AGENT_BRIDGE_READY_TYPE = 'openbitfun:miniapp-agent-ready'; +export const MINIAPP_AGENT_BRIDGE_REQUEST_TYPE = 'openbitfun:miniapp-agent-request'; +export const MINIAPP_AGENT_BRIDGE_RESPONSE_TYPE = 'openbitfun:miniapp-agent-response'; + +export const MINIAPP_AGENT_SURFACE_REQUEST_ACTION = { + open: 'open', + execute: 'execute', +} as const; + +export type MiniAppAgentErrorCode = + | 'INVALID_PARAMS' + | 'UNKNOWN_APP' + | 'UNKNOWN_SESSION' + | 'UNKNOWN_TARGET' + | 'STALE_TARGET' + | 'SURFACE_UNAVAILABLE' + | 'CAPABILITY_MISSING' + | 'BRIDGE_UNAVAILABLE' + | 'BRIDGE_TIMEOUT' + | 'BRIDGE_REJECTED' + | 'TIMEOUT' + | 'HOST_ERROR'; + +/** A typed surface failure. `message` already carries the `CODE: ` prefix. */ +export class MiniAppAgentSurfaceError extends Error { + readonly code: MiniAppAgentErrorCode; + + constructor(code: MiniAppAgentErrorCode, message: string) { + super(`${code}: ${message}`); + this.name = 'MiniAppAgentSurfaceError'; + this.code = code; + } +} + +export function surfaceError(code: MiniAppAgentErrorCode, message: string): MiniAppAgentSurfaceError { + return new MiniAppAgentSurfaceError(code, message); +} + +/** Requests and responses share the same serialized UTF-8 byte contract. */ +export function assertMiniAppAgentBridgePayload( + payload: unknown, + code: MiniAppAgentErrorCode = 'INVALID_PARAMS', +): void { + let serialized: string | undefined; + try { + serialized = JSON.stringify(payload); + } catch { + throw surfaceError(code, 'The bridge payload must be JSON-serializable.'); + } + if (serialized === undefined || new TextEncoder().encode(serialized).byteLength > MINIAPP_AGENT_MAX_BRIDGE_BYTES) { + throw surfaceError(code, 'The bridge payload exceeds the 32 KiB limit.'); + } +} + +/** Locator axes in resolution priority order. */ +export const MINIAPP_AGENT_LOCATOR_PRIORITY = ['testId', 'roleName', 'text', 'selector'] as const; + +export interface MiniAppAgentLocator { + testId?: string; + role?: string; + name?: string; + text?: string; + selector?: string; + exact?: boolean; +} + +/** One observed node. `nodeKey` is host-internal and never reaches the model. */ +export interface MiniAppAgentNode { + nodeKey?: string; + role?: string; + name?: string; + tag?: string; + testId?: string; + value?: string; + text?: string; + enabled?: boolean; + checked?: boolean; + children?: MiniAppAgentNode[]; +} + +export interface MiniAppAgentSurfaceInfo { + appId: string; + instanceId?: string; + strictRuntime: boolean; + width?: number; + height?: number; + capabilities: string[]; + bridgeCapabilities: string[]; + /** Action types declared by the MiniApp's optional agent bridge. */ + declaredActions: string[]; + /** + * Where the app was presented before this request, when the surface had to + * move it. `window` means the app was open in its own window and was + * transferred into the tab the user is watching. + */ + presentedFrom?: 'window' | 'tab'; +} + +export interface MiniAppAgentBridgeAnnouncement { + appId: string; + schemaVersion: string; + capabilities: string[]; + /** Action types the app's `act` handler answers, when it chose to declare them. */ + actions: string[]; +} + +export interface MiniAppAgentBridgeResponse { + requestId: string; + ok: boolean; + value?: unknown; + error?: string; +} + +export function isMiniAppAgentError(error: unknown): error is MiniAppAgentSurfaceError { + return error instanceof MiniAppAgentSurfaceError; +} + +/** Normalize an arbitrary thrown value into the transport's error string. */ +export function miniAppAgentErrorMessage(error: unknown): string { + if (isMiniAppAgentError(error)) return error.message; + if (error instanceof Error) { + return error.message?.trim() ? error.message : 'HOST_ERROR: MiniApp surface action failed'; + } + const text = typeof error === 'string' ? error.trim() : ''; + return text || 'HOST_ERROR: MiniApp surface action failed'; +} + +/** Unique across Web UI reloads; getRandomValues also works on HTTP web surfaces. */ +export function createMiniAppDocumentId(): string { + const bytes = crypto.getRandomValues(new Uint8Array(16)); + return Array.from(bytes, byte => byte.toString(16).padStart(2, '0')).join(''); +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.test.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.test.ts new file mode 100644 index 0000000000..6b8c275a20 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.test.ts @@ -0,0 +1,298 @@ +/** + * @vitest-environment jsdom + */ + +import { afterEach, describe, expect, it } from 'vitest'; + +import { handleMiniAppAgentRequest, type MiniAppAgentRouterDeps } from './miniAppAgentRouter'; +import { + registerMiniAppAgentSurface, + updateMiniAppAgentSurfaceStatus, +} from './miniAppAgentSurfaceRegistry'; +import { MiniAppAgentSurfaceError } from './miniAppAgentProtocol'; + +const HTML = ` +
+

Score

+ + +
+`; + +interface Harness { + deps: MiniAppAgentRouterDeps; + frameWindow: Window; + document: Document; + announce(data: Record): void; +} + +const cleanups: (() => void)[] = []; + +function createHarness(options: { readable?: boolean; ready?: boolean } = {}): Harness { + const doc = document.implementation.createHTMLDocument('Demo'); + doc.body.innerHTML = HTML; + const frameWindow = { + postMessage: (payload: Record) => { + const requestId = String(payload.requestId ?? ''); + queueMicrotask(() => { + const event = new MessageEvent('message', { + data: { + type: 'openbitfun:miniapp-agent-response', + requestId, + ok: true, + // Matches the injected runtime: declared state plus automatic + // observation travel in one envelope. + value: { + state: { echo: payload.method }, + auto: { source: 'declared', canvasText: [], frames: 3 }, + }, + }, + }); + Object.defineProperty(event, 'source', { value: frameWindow }); + window.dispatchEvent(event); + }); + }, + } as unknown as Window; + + const iframe = options.readable === false + ? Object.defineProperty({}, 'contentDocument', { get() { throw new Error('cross-origin'); } }) as unknown as HTMLIFrameElement + : ({ contentDocument: doc, contentWindow: frameWindow, getBoundingClientRect: () => ({ width: 800, height: 600 }) } as unknown as HTMLIFrameElement); + + const deps: MiniAppAgentRouterDeps = { + openTimeoutMs: 200, + presentApp: async () => { + const unregister = registerMiniAppAgentSurface({ + appId: 'demo', + iframe, + strictRuntime: options.readable === false, + instanceId: 'instance-1', + ready: false, + }); + cleanups.push(unregister); + updateMiniAppAgentSurfaceStatus('demo', { ready: options.ready !== false }); + return { instanceId: 'instance-1' }; + }, + }; + + return { + deps, + frameWindow, + document: doc, + announce: (data) => { + const event = new MessageEvent('message', { data }); + Object.defineProperty(event, 'source', { value: frameWindow }); + window.dispatchEvent(event); + }, + }; +} + +function execute(method: string, params: Record = {}) { + return { + requestId: `req-${method}`, + action: 'execute', + command: { appId: 'demo', instanceId: 'instance-1', method, params }, + }; +} + +async function errorOf(run: Promise): Promise { + try { + await run; + } catch (error) { + expect(error).toBeInstanceOf(MiniAppAgentSurfaceError); + return error as MiniAppAgentSurfaceError; + } + throw new Error('expected a typed surface error'); +} + +afterEach(() => { + while (cleanups.length > 0) cleanups.pop()?.(); +}); + +describe('MiniApp agent router', () => { + it('opens the app and reports the capabilities it can actually provide', async () => { + const harness = createHarness(); + // The request carries the host's runtime fact, but the mounted runner owns + // the surface truth, so a readable frame stays readable either way. + const surface = await handleMiniAppAgentRequest( + { requestId: 'req-open', action: 'open', appId: 'demo', strictRuntime: true }, + harness.deps, + ); + expect(surface).toMatchObject({ + appId: 'demo', + instanceId: 'instance-1', + width: 800, + height: 600, + capabilities: ['dom', 'accessibility', 'input'], + bridgeCapabilities: [], + // Nothing was declared yet, so the action vocabulary is unknown, not empty. + declaredActions: [], + }); + }); + + it('reports a presentation transfer so a moved window is visible to the caller', async () => { + const harness = createHarness(); + const deps: MiniAppAgentRouterDeps = { + ...harness.deps, + presentApp: async (appId) => ({ ...(await harness.deps.presentApp(appId)), presentedFrom: 'window' }), + }; + const surface = await handleMiniAppAgentRequest( + { requestId: 'req-open', action: 'open', appId: 'demo' }, + deps, + ); + expect(surface).toMatchObject({ instanceId: 'instance-1', presentedFrom: 'window' }); + }); + + it('omits the transfer marker when the app was already a tab', async () => { + const harness = createHarness(); + const surface = await handleMiniAppAgentRequest( + { requestId: 'req-open', action: 'open', appId: 'demo' }, + harness.deps, + ); + expect(surface).not.toHaveProperty('presentedFrom'); + }); + + it('promises no DOM control for an opaque frame', async () => { const harness = createHarness({ readable: false }); + const surface = await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + expect(surface.capabilities).toEqual([]); + expect(surface.strictRuntime).toBe(true); + }); + + it('routes DOM actions through the surface runtime', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + + const inspect = await handleMiniAppAgentRequest(execute('inspect', { maxNodes: 40, maxDepth: 6 }), harness.deps); + expect(Array.isArray(inspect.tree)).toBe(true); + + const found = await handleMiniAppAgentRequest( + execute('find', { locator: { testId: 'reset' } }), + harness.deps, + ) as { matches: { nodeKey: string }[] }; + expect(found.matches).toHaveLength(1); + + const clicked = await handleMiniAppAgentRequest(execute('click', { nodeKey: found.matches[0].nodeKey }), harness.deps); + expect(clicked).toMatchObject({ target: found.matches[0].nodeKey }); + }); + + it('preserves long text, surrounding whitespace, and empty replacement text', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'open', action: 'open', appId: 'demo' }, harness.deps); + const found = await handleMiniAppAgentRequest(execute('find', { locator: { role: 'textbox' } }), harness.deps) as { matches: { nodeKey: string }[] }; + const nodeKey = found.matches[0].nodeKey; + for (const text of [' padded text ', 'x'.repeat(450), '']) { + await handleMiniAppAgentRequest(execute('type', { nodeKey, text }), harness.deps); + expect(harness.document.querySelector('input')!.value).toBe(text); + await handleMiniAppAgentRequest(execute('type', { nodeKey, text: 'END', clear: false }), harness.deps); + expect(harness.document.querySelector('input')!.value).toBe(`${text}END`); + } + }); + + it('stops an open request when its device activation ends', async () => { + const harness = createHarness({ ready: false }); + const controller = new AbortController(); + const pending = handleMiniAppAgentRequest( + { requestId: 'open', action: 'open', appId: 'demo' }, + { ...harness.deps, signal: controller.signal }, + ); + controller.abort(); + expect((await errorOf(pending)).code).toBe('SURFACE_UNAVAILABLE'); + }); + + it('relays bridge calls and stores the announced capabilities', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + harness.announce({ + type: 'openbitfun:miniapp-agent-ready', + appId: 'demo', + schemaVersion: 'agent.v1', + capabilities: ['observe'], + actions: ['start', 'autoplay'], + }); + + const observed = await handleMiniAppAgentRequest( + execute('bridge_observe', { requestId: 'bridge-1' }), + harness.deps, + ); + expect(observed).toMatchObject({ + state: { echo: 'observe' }, + auto: { source: 'declared' }, + capabilities: ['observe'], + // The vocabulary rides along with every answer so the Agent never has to + // guess an action name or read the app's source to learn one. + actions: ['start', 'autoplay'], + }); + + const missingAct = await errorOf(handleMiniAppAgentRequest( + execute('bridge_act', { requestId: 'bridge-2', action: { type: 'restart' } }), + harness.deps, + )); + expect(missingAct.code).toBe('BRIDGE_UNAVAILABLE'); + }); + + it('refuses a retired sampling-window method instead of half-implementing it', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + + // Sampling is a bounded loop over bridge_observe in the tool, so the surface + // must not grow a second implementation that could drift from it. + const watched = await errorOf(handleMiniAppAgentRequest( + execute('bridge_watch', { requestId: 'watch-1', durationMs: 250, pollMs: 250 }), + harness.deps, + )); + expect(watched.code).toBe('INVALID_PARAMS'); + }); + + it('refuses a stale instance, a missing surface, and unknown methods', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + + const staleInstance = await errorOf(handleMiniAppAgentRequest({ + requestId: 'req-stale', + action: 'execute', + command: { appId: 'demo', instanceId: 'instance-2', method: 'inspect' }, + }, harness.deps)); + expect(staleInstance.code).toBe('SURFACE_UNAVAILABLE'); + + const missing = await errorOf(handleMiniAppAgentRequest({ + requestId: 'req-missing', + action: 'execute', + command: { appId: 'other-app', method: 'inspect' }, + }, harness.deps)); + expect(missing.code).toBe('SURFACE_UNAVAILABLE'); + + const unsupported = await errorOf(handleMiniAppAgentRequest(execute('evaluate'), harness.deps)); + expect(unsupported.code).toBe('INVALID_PARAMS'); + }); + + it('reports unknown actions and missing parameters instead of acting', async () => { + const harness = createHarness(); + await handleMiniAppAgentRequest({ requestId: 'req-open', action: 'open', appId: 'demo' }, harness.deps); + + expect((await errorOf(handleMiniAppAgentRequest({ requestId: 'req-1', action: 'nope' }, harness.deps))).code) + .toBe('INVALID_PARAMS'); + expect((await errorOf(handleMiniAppAgentRequest({ action: 'execute' }, harness.deps))).code) + .toBe('INVALID_PARAMS'); + expect((await errorOf(handleMiniAppAgentRequest(execute('click'), harness.deps))).code) + .toBe('INVALID_PARAMS'); + }); + + it('rejects existing-session commands while its frame is not ready', async () => { + const harness = createHarness({ ready: false }); + await harness.deps.presentApp('demo'); + for (const method of ['inspect', 'find', 'key', 'bridge_act']) { + const error = await errorOf(handleMiniAppAgentRequest(execute(method), harness.deps)); + expect(error.code).toBe('SURFACE_UNAVAILABLE'); + } + updateMiniAppAgentSurfaceStatus('demo', { ready: true }); + await expect(handleMiniAppAgentRequest(execute('inspect'), harness.deps)).resolves.toHaveProperty('tree'); + }); + + it('times out when the presented app never becomes controllable', async () => { + const harness = createHarness({ ready: false }); + const error = await errorOf(handleMiniAppAgentRequest( + { requestId: 'req-open', action: 'open', appId: 'demo' }, + { ...harness.deps, openTimeoutMs: 50 }, + )); + expect(error.code).toBe('TIMEOUT'); + }); +}); diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.ts new file mode 100644 index 0000000000..e811b5ff4e --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentRouter.ts @@ -0,0 +1,282 @@ +/** + * Routes one host request to the right surface runtime. + * + * This is the seam between the transport (a native event plus a result command) + * and the surface capabilities (DOM runtime, bridge client, presentation). It + * holds no React state and no Tauri calls, so both the hook and a plain test can + * drive it with explicit dependencies. + */ + +import { + canReadSurfaceDocument, + clickNode, + findNodes, + inspectDocument, + pressKey, + readNode, + scrollSurface, + typeIntoNode, + type MiniAppAgentDomSurface, +} from './miniAppAgentDomRuntime'; +import { captureSurface } from './miniAppAgentCapture'; +import { + getMiniAppAgentSurface, + updateMiniAppAgentSurfaceStatus, + waitForMiniAppAgentSurface, + type MiniAppAgentSurfaceEntry, +} from './miniAppAgentSurfaceRegistry'; +import { + MINIAPP_AGENT_DEFAULT_CAPTURE_EDGE, + MINIAPP_AGENT_MAX_NAME_CHARS, + MINIAPP_AGENT_MAX_TEXT_CHARS, + MINIAPP_AGENT_SURFACE_REQUEST_ACTION, + surfaceError, + type MiniAppAgentLocator, + type MiniAppAgentSurfaceInfo, +} from './miniAppAgentProtocol'; + +/** Default budget for mounting a tab and its frame. */ +export const MINIAPP_AGENT_OPEN_TIMEOUT_MS = 60_000; + +export interface MiniAppAgentRequestPayload { + requestId: string; + action: string; + appId?: string; + name?: string; + strictRuntime?: boolean; + command?: { + appId: string; + instanceId?: string; + method: string; + params?: Record | null; + }; +} + +export interface MiniAppAgentRouterDeps { + /** + * Present the app in a surface this window can read. Must throw a typed + * surface error when the app is shown somewhere the controller cannot reach. + */ + presentApp(appId: string): Promise<{ instanceId?: string; presentedFrom?: 'window' | 'tab' }>; + openTimeoutMs?: number; + signal?: AbortSignal; +} + +function readString(source: Record, key: string, limit = MINIAPP_AGENT_MAX_NAME_CHARS): string | undefined { + const value = source[key]; + if (typeof value !== 'string') return undefined; + const trimmed = value.trim(); + return trimmed ? trimmed.slice(0, limit) : undefined; +} + +function requireString(source: Record, key: string, action: string): string { + const value = readString(source, key, MINIAPP_AGENT_MAX_TEXT_CHARS); + if (!value) throw surfaceError('INVALID_PARAMS', `${action} requires '${key}'.`); + return value; +} + +function requireNodeKey(source: Record, action: string): string { + const value = requireString(source, 'nodeKey', action); + return value; +} + +function numberParam(source: Record, key: string): number | undefined { + const value = source[key]; + if (typeof value !== 'number' || !Number.isFinite(value)) return undefined; + return value; +} + +function sanitizeLocator(value: unknown): MiniAppAgentLocator { + const source = (value && typeof value === 'object' ? value : {}) as Record; + const locator: MiniAppAgentLocator = {}; + const testId = readString(source, 'testId'); + if (testId) locator.testId = testId; + const role = readString(source, 'role', 40); + if (role) locator.role = role.toLowerCase(); + const name = readString(source, 'name'); + if (name) locator.name = name; + const text = readString(source, 'text', MINIAPP_AGENT_MAX_TEXT_CHARS); + if (text) locator.text = text; + const selector = readString(source, 'selector'); + if (selector) locator.selector = selector; + if (source.exact === true) locator.exact = true; + return locator; +} + +function parseRequest(raw: unknown): MiniAppAgentRequestPayload { + const source = (raw && typeof raw === 'object' ? raw : {}) as Record; + const requestId = readString(source, 'requestId', 128); + if (!requestId) { + throw surfaceError('INVALID_PARAMS', 'The host request is missing its requestId.'); + } + const action = readString(source, 'action', 32) ?? ''; + const payload: MiniAppAgentRequestPayload = { + requestId, + action, + appId: readString(source, 'appId'), + name: readString(source, 'name'), + strictRuntime: source.strictRuntime === true, + }; + const command = source.command as Record | undefined; + if (command && typeof command === 'object') { + const appId = readString(command, 'appId'); + const method = readString(command, 'method', 40); + if (appId && method) { + payload.command = { + appId, + instanceId: readString(command, 'instanceId', 128), + method, + params: (command.params && typeof command.params === 'object' + ? command.params + : {}) as Record, + }; + } + } + return payload; +} + +function requireSurface(appId: string, instanceId?: string): MiniAppAgentSurfaceEntry { + const entry = getMiniAppAgentSurface(appId); + if (!entry || !entry.ready) { + throw surfaceError( + 'SURFACE_UNAVAILABLE', + `The MiniApp '${appId}' is not ready in this window. Wait for it to load or open it again from this surface.`, + ); + } + if (instanceId && entry.instanceId && instanceId !== entry.instanceId) { + throw surfaceError( + 'SURFACE_UNAVAILABLE', + `The MiniApp '${appId}' is now running as a different instance. Open it again before acting.`, + ); + } + return entry; +} + +function surfaceSize(entry: MiniAppAgentSurfaceEntry): { width?: number; height?: number } { + const rect = entry.iframe.getBoundingClientRect?.(); + const width = rect && rect.width > 0 ? Math.round(rect.width) : undefined; + const height = rect && rect.height > 0 ? Math.round(rect.height) : undefined; + return { width, height }; +} + +async function openSurface( + payload: MiniAppAgentRequestPayload, + deps: MiniAppAgentRouterDeps, +): Promise { + const appId = payload.appId; + if (!appId) { + throw surfaceError('INVALID_PARAMS', 'open requires appId.'); + } + const presented = await deps.presentApp(appId); + const entry = await waitForMiniAppAgentSurface(appId, deps.openTimeoutMs ?? MINIAPP_AGENT_OPEN_TIMEOUT_MS, deps.signal); + if (presented.instanceId) { + updateMiniAppAgentSurfaceStatus(appId, { instanceId: presented.instanceId }); + } + + // A readable frame is the only reason to promise DOM control. Anything else + // must be answered honestly instead of guessed from the runtime profile. + const readable = canReadSurfaceDocument(entry); + const { width, height } = surfaceSize(entry); + return { + appId, + instanceId: presented.instanceId ?? entry.instanceId, + strictRuntime: entry.strictRuntime, + width, + height, + capabilities: readable ? ['dom', 'accessibility', 'input'] : [], + bridgeCapabilities: entry.bridge.capabilities, + declaredActions: entry.bridge.actions, + ...(presented.presentedFrom ? { presentedFrom: presented.presentedFrom } : {}), + }; +} + +function domSurface(entry: MiniAppAgentSurfaceEntry): MiniAppAgentDomSurface { + return { appId: entry.appId, iframe: entry.iframe }; +} + +async function executeSurface( + payload: MiniAppAgentRequestPayload, +): Promise> { + const command = payload.command; + if (!command) { + throw surfaceError('INVALID_PARAMS', 'execute requires a command.'); + } + const entry = requireSurface(command.appId, command.instanceId); + const params = command.params ?? {}; + const dom = domSurface(entry); + + switch (command.method) { + case 'inspect': + return inspectDocument(dom, { + maxNodes: numberParam(params, 'maxNodes'), + maxDepth: numberParam(params, 'maxDepth'), + }) as unknown as Record; + case 'find': + return { matches: findNodes(dom, sanitizeLocator(params.locator)) }; + case 'read': + return readNode(dom, requireNodeKey(params, 'read'), readString(params, 'attribute', 80)) as unknown as Record; + case 'click': + return clickNode(dom, requireNodeKey(params, 'click')) as unknown as Record; + case 'type': + if (typeof params.text !== 'string') { + throw surfaceError('INVALID_PARAMS', "type requires 'text' to be a string."); + } + return typeIntoNode( + dom, + requireNodeKey(params, 'type'), + params.text, + params.clear !== false, + ) as unknown as Record; + case 'key': + return (await pressKey(dom, requireString(params, 'key', 'key'), { + modifiers: Array.isArray(params.modifiers) + ? params.modifiers.filter((modifier): modifier is string => typeof modifier === 'string') + : [], + holdMs: numberParam(params, 'holdMs') ?? 0, + })) as unknown as Record; + case 'scroll': + return scrollSurface( + dom, + readString(params, 'nodeKey', 128), + numberParam(params, 'deltaX') ?? 0, + numberParam(params, 'deltaY') ?? 0, + ); + case 'capture': + return (await captureSurface( + dom, + numberParam(params, 'maxEdge') ?? MINIAPP_AGENT_DEFAULT_CAPTURE_EDGE, + )) as unknown as Record; + case 'bridge_observe': + return entry.bridge.observe(requireString(params, 'requestId', 'bridge_observe')); + case 'bridge_act': + return entry.bridge.act( + requireString(params, 'requestId', 'bridge_act'), + params.action, + ); + case 'bridge_watch': + throw surfaceError( + 'INVALID_PARAMS', + 'bridge_watch is no longer a surface method: sampling is a bounded loop over bridge_observe in the tool.'); + default: + throw surfaceError('INVALID_PARAMS', `Unsupported MiniApp surface method: ${command.method}`); + } +} + +/** Handle one host request and return the value to report back. */ +export async function handleMiniAppAgentRequest( + raw: unknown, + deps: MiniAppAgentRouterDeps, +): Promise> { + if (deps.signal?.aborted) { + throw surfaceError('SURFACE_UNAVAILABLE', 'The MiniApp device surface is no longer active.'); + } + const payload = parseRequest(raw); + switch (payload.action) { + case MINIAPP_AGENT_SURFACE_REQUEST_ACTION.open: + return openSurface(payload, deps) as unknown as Record; + case MINIAPP_AGENT_SURFACE_REQUEST_ACTION.execute: + return executeSurface(payload); + default: + throw surfaceError('INVALID_PARAMS', `Unsupported MiniApp surface action: ${payload.action || '(empty)'}`); + } +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentSurfaceRegistry.ts b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentSurfaceRegistry.ts new file mode 100644 index 0000000000..f67bc5c622 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/miniAppAgentSurfaceRegistry.ts @@ -0,0 +1,96 @@ +/** + * Live MiniApp surfaces that the Agent may control. + * + * The registry is the only place that knows which mounted frame belongs to + * which app. `MiniAppRunner` registers on mount and unregisters on unmount, so a + * closed tab, a reload, or a transfer to a separate window removes the surface + * instead of leaving a stale handle behind. + */ + +import { MiniAppAgentBridge, type MiniAppAgentBridgeSurface } from './miniAppAgentBridge'; +import { surfaceError } from './miniAppAgentProtocol'; + +export interface MiniAppAgentSurfaceRegistration extends MiniAppAgentBridgeSurface { + strictRuntime: boolean; + instanceId?: string; + ready: boolean; +} + +export interface MiniAppAgentSurfaceEntry extends MiniAppAgentSurfaceRegistration { + bridge: MiniAppAgentBridge; +} + +const surfaces = new Map(); + +const WAIT_POLL_MS = 50; + +/** Register one mounted frame. Returns an idempotent unregister function. */ +export function registerMiniAppAgentSurface( + registration: MiniAppAgentSurfaceRegistration, +): () => void { + const previous = surfaces.get(registration.appId); + if (previous && previous.iframe !== registration.iframe) { + // A new frame replaced the old one; its handles must not answer for it. + previous.bridge.stop(); + } + const entry: MiniAppAgentSurfaceEntry = { + ...registration, + bridge: previous && previous.iframe === registration.iframe ? previous.bridge : new MiniAppAgentBridge(registration), + }; + entry.bridge.start(); + surfaces.set(registration.appId, entry); + return () => { + const current = surfaces.get(registration.appId); + if (current !== entry) return; + surfaces.delete(registration.appId); + entry.bridge.stop(); + }; +} + +/** Mount-time readiness is separate from registration, so it can change freely. */ +export function updateMiniAppAgentSurfaceStatus( + appId: string, + patch: { ready?: boolean; instanceId?: string }, +): void { + const entry = surfaces.get(appId); + if (!entry) return; + if (patch.ready !== undefined) entry.ready = patch.ready; + if (patch.instanceId !== undefined) entry.instanceId = patch.instanceId; +} + +export function getMiniAppAgentSurface(appId: string): MiniAppAgentSurfaceEntry | undefined { + return surfaces.get(appId); +} + +export function listMiniAppAgentSurfaces(): MiniAppAgentSurfaceEntry[] { + return [...surfaces.values()]; +} + +/** Drop every surface; used when the surface host itself shuts down. */ +export function releaseMiniAppAgentSurfaces(): void { + for (const entry of surfaces.values()) entry.bridge.stop(); + surfaces.clear(); +} + +/** Wait for the frame the Agent asked to open. Never resolves to a guess. */ +export async function waitForMiniAppAgentSurface( + appId: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + const deadline = Date.now() + Math.max(0, timeoutMs); + for (;;) { + if (signal?.aborted) { + throw surfaceError('SURFACE_UNAVAILABLE', 'The MiniApp device surface is no longer active.'); + } + const entry = surfaces.get(appId); + if (entry?.ready) return entry; + if (Date.now() >= deadline) { + throw surfaceError( + 'TIMEOUT', + `The MiniApp '${appId}' surface did not become ready within ${timeoutMs}ms.`, + ); + } + await new Promise((resolve) => setTimeout(resolve, WAIT_POLL_MS)); + } +} diff --git a/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.test.tsx b/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.test.tsx new file mode 100644 index 0000000000..47e6585ffb --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.test.tsx @@ -0,0 +1,109 @@ +/** @vitest-environment jsdom */ + +import React, { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { activateSurface, getActiveSurfaceScope, resetDeviceSurfaceForTest } from '@/infrastructure/peer-device/deviceSurface'; +import { useMiniAppAgentSurface } from './useMiniAppAgentSurface'; + +const mocks = vi.hoisted(() => ({ + handlers: new Set<(request: unknown) => void>(), + markReady: vi.fn(async (_ready: boolean) => undefined), + reportResult: vi.fn(async (_report: unknown) => undefined), + handleRequest: vi.fn(), + release: vi.fn(), +})); + +vi.mock('@/infrastructure/api/service-api/MiniAppPresentationAPI', async () => { + const { getActiveSurfaceId } = await import('@/infrastructure/peer-device/deviceSurface'); + return { hasLocalMiniAppPresentation: () => getActiveSurfaceId() === 'local' }; +}); +vi.mock('@/infrastructure/api/service-api/MiniAppAgentSurfaceAPI', () => ({ + miniAppAgentSurfaceAPI: { + markReady: mocks.markReady, + reportResult: mocks.reportResult, + listenRequests: (handler: (request: unknown) => void) => { + mocks.handlers.add(handler); + return () => mocks.handlers.delete(handler); + }, + }, +})); +vi.mock('@/app/stores/sceneStore', () => ({ useSceneStore: { getState: () => ({ openScene: vi.fn() }) } })); +vi.mock('../miniAppPresentationStore', () => ({ + ensureMiniAppTab: vi.fn(), + useMiniAppPresentationStore: { getState: () => ({ tabs: {} }) }, +})); +vi.mock('./miniAppAgentSurfaceRegistry', () => ({ releaseMiniAppAgentSurfaces: mocks.release })); +vi.mock('./miniAppAgentRouter', () => ({ + MINIAPP_AGENT_OPEN_TIMEOUT_MS: 60_000, + handleMiniAppAgentRequest: mocks.handleRequest, +})); + +function Surface() { + useMiniAppAgentSurface(); + return null; +} + +describe('MiniApp Agent device surface lifecycle', () => { + let container: HTMLDivElement; + let root: Root; + + beforeEach(() => { + resetDeviceSurfaceForTest(); + mocks.handlers.clear(); + vi.clearAllMocks(); + mocks.handleRequest.mockResolvedValue({ ok: true }); + container = document.createElement('div'); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + resetDeviceSurfaceForTest(); + }); + + it('detaches locally on Peer activation and reattaches on return', async () => { + await act(async () => root.render()); + expect(mocks.markReady).toHaveBeenLastCalledWith(true); + expect(mocks.handlers.size).toBe(1); + await act(async () => { activateSurface('peer-b'); }); + expect(mocks.markReady).toHaveBeenLastCalledWith(false); + expect(mocks.handlers.size).toBe(0); + expect(mocks.release).toHaveBeenCalled(); + await act(async () => { activateSurface('local'); }); + expect(mocks.markReady).toHaveBeenLastCalledWith(true); + expect(mocks.handlers.size).toBe(1); + await act(async () => { + for (const handler of mocks.handlers) handler({ requestId: 'returned', action: 'open', appId: 'demo' }); + }); + expect(mocks.reportResult).toHaveBeenCalledWith({ requestId: 'returned', success: true, result: { ok: true } }); + }); + + it('fences an in-flight local response after switching to a Peer', async () => { + let finish!: (value: unknown) => void; + mocks.handleRequest.mockImplementation(() => new Promise(resolve => { finish = resolve; })); + await act(async () => root.render()); + const originalScope = getActiveSurfaceScope(); + await act(async () => { + for (const handler of mocks.handlers) handler({ requestId: 'old', action: 'open', appId: 'demo' }); + }); + expect(mocks.handleRequest.mock.calls[0][1].signal).toBe(originalScope.signal); + await act(async () => { activateSurface('peer-b'); }); + expect(originalScope.signal.aborted).toBe(true); + await act(async () => { finish({ wrongDevice: true }); }); + expect(mocks.reportResult).toHaveBeenCalledWith(expect.objectContaining({ + requestId: 'old', success: false, error: expect.stringContaining('SURFACE_UNAVAILABLE:'), + })); + expect(mocks.reportResult).not.toHaveBeenCalledWith(expect.objectContaining({ success: true })); + }); + + it('keeps mounted local frames when only the activation epoch changes', async () => { + await act(async () => root.render()); + await act(async () => { activateSurface('local'); }); + expect(mocks.markReady).toHaveBeenLastCalledWith(true); + expect(mocks.handlers.size).toBe(1); + expect(mocks.release).not.toHaveBeenCalled(); + }); +}); diff --git a/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.ts b/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.ts new file mode 100644 index 0000000000..3a8bc1abf4 --- /dev/null +++ b/src/web-ui/src/app/scenes/miniapps/agent/useMiniAppAgentSurface.ts @@ -0,0 +1,119 @@ +/** + * Mount point for the Agent-controlled MiniApp surface. + * + * Exactly one instance exists in the main window. It registers the window as a + * surface listener, presents the requested app in a tab, and answers every + * request with either a result or a typed error. It never runs on a device + * surface that cannot present a local MiniApp, and it never pretends a request + * succeeded when the frame is unreachable. + */ + +import { useCallback, useEffect, useSyncExternalStore } from 'react'; +import { getActiveSurfaceScope, onSurfaceActivated } from '@/infrastructure/peer-device/deviceSurface'; +import { hasLocalMiniAppPresentation } from '@/infrastructure/api/service-api/MiniAppPresentationAPI'; +import { miniAppAgentSurfaceAPI } from '@/infrastructure/api/service-api/MiniAppAgentSurfaceAPI'; +import { useSceneStore } from '@/app/stores/sceneStore'; +import type { SceneTabId } from '@/app/components/SceneBar/types'; +import { createLogger } from '@/shared/utils/logger'; +import { + MINIAPP_AGENT_OPEN_TIMEOUT_MS, + handleMiniAppAgentRequest, + type MiniAppAgentRouterDeps, +} from './miniAppAgentRouter'; +import { ensureMiniAppTab, useMiniAppPresentationStore } from '../miniAppPresentationStore'; +import { miniAppAgentErrorMessage, surfaceError } from './miniAppAgentProtocol'; +import { releaseMiniAppAgentSurfaces } from './miniAppAgentSurfaceRegistry'; + +const log = createLogger('MiniAppAgentSurface'); + +/** + * Present the app where the controller can read it, or fail honestly. + * + * A MiniApp can be reached from wherever the user opened it. A tab is already + * readable; an app running in its own window is a separate WebView this window + * cannot inspect, so the host transfers that live instance into a tab of the + * window the user is watching. The Agent then drives the same instance the user + * sees, instead of a second copy or an invisible headless one. + */ +async function presentApp( + appId: string, +): Promise<{ instanceId?: string; presentedFrom?: 'window' | 'tab' }> { + if (!hasLocalMiniAppPresentation()) { + throw surfaceError( + 'SURFACE_UNAVAILABLE', + 'This device surface cannot present a MiniApp locally, so the Agent cannot control it from here.', + ); + } + const scope = getActiveSurfaceScope(); + const store = useMiniAppPresentationStore.getState(); + const existing = store.tabs[appId]; + const inSeparateWindow = !existing && store.snapshot?.instances.some( + (instance) => instance.appId === appId && instance.primaryView?.kind === 'window', + ); + if (inSeparateWindow) { + log.debug('Transferring MiniApp window presentation into an Agent-readable tab', { appId }); + } + const ticket = existing ?? await ensureMiniAppTab(appId); + scope.assertCurrent('present MiniApp for Agent'); + if (ticket.view.kind !== 'tab') { + throw surfaceError( + 'SURFACE_UNAVAILABLE', + `The MiniApp '${appId}' is presented as a ${ticket.view.kind}, which this surface cannot read.`, + ); + } + useSceneStore.getState().openScene(`miniapp:${appId}` as SceneTabId); + return { + instanceId: ticket.instance.instanceId, + ...(inSeparateWindow ? { presentedFrom: 'window' as const } : {}), + }; +} + +export function useMiniAppAgentSurface(): void { + const scope = useSyncExternalStore(onSurfaceActivated, getActiveSurfaceScope, getActiveSurfaceScope); + useEffect(() => () => releaseMiniAppAgentSurfaces(), []); + const handleRequest = useCallback(async (request: unknown) => { + const deps: MiniAppAgentRouterDeps = { + presentApp, + openTimeoutMs: MINIAPP_AGENT_OPEN_TIMEOUT_MS, + signal: scope.signal, + }; + const requestId = (request as { requestId?: unknown })?.requestId; + if (typeof requestId !== 'string' || !requestId) return; + try { + if (!scope.isCurrent() || !hasLocalMiniAppPresentation()) { + throw surfaceError('SURFACE_UNAVAILABLE', 'The local MiniApp surface is not active.'); + } + const result = await handleMiniAppAgentRequest(request, deps); + scope.assertCurrent('complete MiniApp Agent request'); + await miniAppAgentSurfaceAPI.reportResult({ requestId, success: true, result }); + } catch (error) { + const message = scope.isCurrent() ? miniAppAgentErrorMessage(error) + : surfaceError('SURFACE_UNAVAILABLE', 'The local MiniApp device surface is no longer active.').message; + log.debug('MiniApp surface request failed', { requestId, error: message }); + await miniAppAgentSurfaceAPI + .reportResult({ requestId, success: false, error: message }) + .catch((reportError) => log.debug('Failed to report MiniApp surface failure', { reportError })); + } + }, [scope]); + + useEffect(() => { + let disposed = false; + const ready = hasLocalMiniAppPresentation(); + // An epoch may change during a reconnect on the same device. Keep mounted + // frames in that case; only leaving local presentation retires them. + if (!ready) releaseMiniAppAgentSurfaces(); + const unlisten = ready ? miniAppAgentSurfaceAPI.listenRequests((request) => { + if (disposed) return; + void handleRequest(request); + }) : () => undefined; + // Only a surface that can actually present a local MiniApp may answer. + void miniAppAgentSurfaceAPI.markReady(ready).catch((error) => { + log.debug('MiniApp agent surface unavailable', { error }); + }); + return () => { + disposed = true; + unlisten(); + void miniAppAgentSurfaceAPI.markReady(false).catch(() => undefined); + }; + }, [handleRequest]); +} diff --git a/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.test.tsx b/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.test.tsx index c57839c04b..fa4a2b6ee1 100644 --- a/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.test.tsx +++ b/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.test.tsx @@ -9,12 +9,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import type { MiniApp } from '@/infrastructure/api/service-api/MiniAppAPI'; import MiniAppRunner from './MiniAppRunner'; +import { getMiniAppAgentSurface } from '../agent/miniAppAgentSurfaceRegistry'; const bridgeMock = vi.hoisted(() => ({ ready: false, useMiniAppBridge: vi.fn(), })); +vi.mock('../miniAppPresentationStore', () => ({ + useMiniAppPresentationStore: (selector: (state: unknown) => unknown) => selector({ tabs: {} }), +})); + vi.mock('../hooks/useMiniAppBridge', () => ({ useMiniAppBridge: (...args: unknown[]) => { bridgeMock.useMiniAppBridge(...args); @@ -85,6 +90,9 @@ describe('MiniAppRunner strict bridge startup', () => { expect(iframe.getAttribute('src')).toBe('blob:market-lens-1'); expect(iframe.getAttribute('srcdoc')).toBeNull(); expect(iframe.getAttribute('sandbox')).toBe('allow-scripts allow-downloads'); + expect(getMiniAppAgentSurface(app.id)).toMatchObject({ iframe, ready: false }); + await act(async () => { iframe.dispatchEvent(new Event('load')); }); + expect(getMiniAppAgentSurface(app.id)).toMatchObject({ iframe, ready: true }); const updatedApp = { ...app, @@ -97,6 +105,9 @@ describe('MiniAppRunner strict bridge startup', () => { expect(revokeObjectURL).toHaveBeenCalledWith('blob:market-lens-1'); iframe = container.querySelector('iframe') as HTMLIFrameElement; expect(iframe.getAttribute('src')).toBe('blob:market-lens-2'); + expect(getMiniAppAgentSurface(app.id)).toMatchObject({ iframe, ready: false }); + await act(async () => { iframe.dispatchEvent(new Event('load')); }); + expect(getMiniAppAgentSurface(app.id)).toMatchObject({ iframe, ready: true }); await act(async () => { root?.unmount(); @@ -104,6 +115,34 @@ describe('MiniAppRunner strict bridge startup', () => { root = null; expect(revokeObjectURL).toHaveBeenCalledWith('blob:market-lens-2'); }); + it('retires the old document and bridge when ordinary compiled HTML changes', async () => { + bridgeMock.ready = true; + await act(async () => { root?.render(); }); + const oldFrame = container.querySelector('iframe')!; + const oldWindow = oldFrame.contentWindow!; + const oldEntry = getMiniAppAgentSurface(app.id)!; + const announce = (source: Window, capabilities: string[]) => window.dispatchEvent(new MessageEvent('message', { + source, data: { type: 'openbitfun:miniapp-agent-ready', appId: app.id, schemaVersion: 'agent.v1', capabilities }, + })); + announce(oldWindow, ['act']); + const pending = oldEntry.bridge.act('old-document', { type: 'save' }); + const retired = expect(pending).rejects.toThrow('SURFACE_UNAVAILABLE'); + await act(async () => { root?.render(Replacement' }} />); }); + await retired; + const nextFrame = container.querySelector('iframe')!; + const nextEntry = getMiniAppAgentSurface(app.id)!; + expect(nextFrame).not.toBe(oldFrame); + expect(oldFrame.isConnected).toBe(false); + expect(nextEntry.bridge).not.toBe(oldEntry.bridge); + expect(nextEntry.bridge.announced).toBe(false); + // Queued messages from the retired Window cannot declare or complete work + // for the new frame, even with the same app and request ids. + announce(oldWindow, ['act']); + expect(nextEntry.bridge.announced).toBe(false); + announce(nextFrame.contentWindow!, ['observe']); + expect(nextEntry.bridge.capabilities).toEqual(['observe']); + }); + it('delivers only declared enabled chrome actions to its own iframe and releases the binding', async () => { const ref = { current: null as ((actionId: string) => void) | null }; const customApp = { ...app, chrome: { titlebar: { actions: [ diff --git a/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.tsx b/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.tsx index 6ee9937d70..882db05dc6 100644 --- a/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.tsx +++ b/src/web-ui/src/app/scenes/miniapps/components/MiniAppRunner.tsx @@ -3,10 +3,16 @@ * Injects the bridge script (already in compiledHtml from Rust compiler) * and handles all postMessage RPC via useMiniAppBridge. */ -import React, { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react'; +import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'; import type { MiniApp } from '@/infrastructure/api/service-api/MiniAppAPI'; +import { createMiniAppDocumentId } from '../agent/miniAppAgentProtocol'; import { useMiniAppBridge } from '../hooks/useMiniAppBridge'; import type { MiniAppRunScope } from '../customization/miniAppCustomizationTypes'; +import { useMiniAppPresentationStore } from '../miniAppPresentationStore'; +import { + registerMiniAppAgentSurface, + updateMiniAppAgentSurfaceStatus, +} from '../agent/miniAppAgentSurfaceRegistry'; interface MiniAppRunnerProps { app: MiniApp; @@ -18,7 +24,22 @@ interface MiniAppRunnerProps { } const MiniAppRunner: React.FC = ({ app, runScope, strictRuntime = false, workspacePath, onReady, chromeActionRef }) => { - const iframeRef = useRef(null); + const iframeRef = useRef(null); + // Content replacement ends the old document's bridge and pending requests, + // including in the ordinary same-origin runtime. A new Window identity also + // prevents queued messages from the old document answering for the new one. + const documentKey = useMemo(() => createMiniAppDocumentId(), [app.id, app.compiled_html]); + // The strict runtime replaces the frame element when its blob URL changes, so + // Agent registration must follow the mounted element instead of the first + // render; otherwise the registry keeps a detached frame that answers nothing. + const [surfaceFrame, setSurfaceFrame] = useState(null); + const [loadedFrame, setLoadedFrame] = useState(null); + const attachFrame = useCallback((node: HTMLIFrameElement | null) => { + iframeRef.current = node; + setSurfaceFrame(node); + }, []); + const runKind = runScope?.kind ?? 'active'; + const agentInstanceId = useMiniAppPresentationStore(state => state.tabs[app.id]?.instance.instanceId); const bridgeReady = useMiniAppBridge( iframeRef, app, @@ -26,6 +47,29 @@ const MiniAppRunner: React.FC = ({ app, runScope, strictRunt strictRuntime, workspacePath, ); + + // Only the user-visible installed runner is Agent-controllable. A draft + // preview shares the app id but is a different surface, and unmounting must + // remove the surface so no stale handle answers for a closed tab. + useEffect(() => { + if (runKind !== 'active' || !surfaceFrame) return undefined; + return registerMiniAppAgentSurface({ + appId: app.id, + iframe: surfaceFrame, + strictRuntime, + instanceId: agentInstanceId, + ready: false, + }); + }, [agentInstanceId, app.id, runKind, strictRuntime, surfaceFrame]); + + useEffect(() => { + if (runKind !== 'active') return; + updateMiniAppAgentSurfaceStatus(app.id, { + ready: bridgeReady && surfaceFrame !== null && loadedFrame === surfaceFrame, + instanceId: agentInstanceId, + }); + }, [agentInstanceId, app.id, bridgeReady, loadedFrame, runKind, surfaceFrame]); + useEffect(() => { if (!chromeActionRef) return; const invoke = (actionId: string) => { @@ -39,6 +83,7 @@ const MiniAppRunner: React.FC = ({ app, runScope, strictRunt const onReadyRef = useRef(onReady); onReadyRef.current = onReady; const notifyReady = useCallback(() => { + setLoadedFrame(iframeRef.current); if (!notified.current) { notified.current = true; onReadyRef.current?.(); } }, []); const [strictDocumentUrl, setStrictDocumentUrl] = useState(); @@ -65,7 +110,7 @@ const MiniAppRunner: React.FC = ({ app, runScope, strictRunt useEffect(() => { const iframe = iframeRef.current; - if (strictRuntime || !iframe) { + if (strictRuntime || !iframe || iframe !== surfaceFrame) { return undefined; } @@ -90,7 +135,7 @@ const MiniAppRunner: React.FC = ({ app, runScope, strictRunt return () => { iframe.removeEventListener('load', handleLoad); }; - }, [app.id, app.compiled_html, strictRuntime, writeCompiledHtml]); + }, [app.id, app.compiled_html, strictRuntime, surfaceFrame, writeCompiledHtml]); // A fast document may finish before the asynchronous bridge becomes ready. useEffect(() => { @@ -123,7 +168,7 @@ const MiniAppRunner: React.FC = ({ app, runScope, strictRunt return (