From 37344d5a8b9c1d756325b2f5b1613423bf418778 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 21:19:35 -0300 Subject: [PATCH] feat(desktop): the bot's cursor on its desktop panel, which opens by itself Each desktop action now says where it happened in the window: the middle of the control through accessibility, or the point of a desktop_click_at, as fractions of the window's frame (DesktopAction x and y). The desktop panel shows the window on the same stage as the browser, with the bot's cursor gliding there and a ring on each click. The panel also opens by itself when the bot starts using the desktop after two minutes without it, like the browser does, unless the owner turned off following the bot. --- app/src/dev/seedDesktop.ts | 8 +++- app/src/features/bots/BotView.tsx | 26 ++++++---- app/src/features/bots/useFollowBot.ts | 37 ++++++++++++-- app/src/features/browser/LiveView.tsx | 36 +++++++++++--- app/src/features/desktop/DesktopPanel.tsx | 48 ++++++++++++++----- app/src/features/desktop/panel.test.tsx | 42 +++++++++++++++- app/src/i18n/en/desktop.ts | 1 + app/src/i18n/es/desktop.ts | 1 + app/src/i18n/pt-BR/desktop.ts | 1 + app/src/lib/protocol.gen.ts | 8 +++- crates/botloft-core/src/protocol/desktop.rs | 11 +++-- crates/botloftd/src/desktop/activity.rs | 2 + crates/botloftd/src/platform/desktop/mod.rs | 14 ++++++ .../src/platform/desktop/windows/capture.rs | 26 ++++++++++ .../src/platform/desktop/windows/mod.rs | 2 +- crates/botloftd/src/tools/desktop_act.rs | 35 +++++++++++++- crates/botloftd/src/tools/desktop_real.rs | 10 ++-- crates/botloftd/tests/desktop_panel.rs | 23 ++++++++- docs/spec.md | 4 +- 19 files changed, 288 insertions(+), 47 deletions(-) diff --git a/app/src/dev/seedDesktop.ts b/app/src/dev/seedDesktop.ts index 0ce7e2a5..3674baa2 100644 --- a/app/src/dev/seedDesktop.ts +++ b/app/src/dev/seedDesktop.ts @@ -34,7 +34,13 @@ function drawNotepad(): string { export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) { fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad"); - fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null }); + fake.desktop.use(bots.scout, NOTES, { + kind: "click", + target: "Save", + option: null, + x: 0.1, + y: 0.12, + }); fake.desktop.paint(bots.scout, drawNotepad(), 760, 480); fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", { summary: "To save the list", diff --git a/app/src/features/bots/BotView.tsx b/app/src/features/bots/BotView.tsx index 83559fdf..44e52f97 100644 --- a/app/src/features/bots/BotView.tsx +++ b/app/src/features/bots/BotView.tsx @@ -121,15 +121,6 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { setTakeBrowser((count) => count + 1); } }); - // The panel follows what the bot starts doing (spec 15.1), over the one - // that came back too. - useFollowBot({ - bot, - side, - writing: screens.writing, - open: (panel) => (panel === "browser" ? showBrowser() : showScreen(null)), - close: (panel) => side === panel && setSide(null), - }); const showTerminal = useStable(() => { if (filesOpen) { seen(); @@ -142,6 +133,23 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { } setSide("desktop"); }); + // The panel follows what the bot starts doing (spec 15.1), over the one + // that came back too. + useFollowBot({ + bot, + side, + writing: screens.writing, + open: (panel) => { + if (panel === "browser") { + showBrowser(); + } else if (panel === "desktop") { + showDesktop(); + } else { + showScreen(null); + } + }, + close: (panel) => side === panel && setSide(null), + }); // The dock of the bot's computer: browser, terminal and files. const dock = { open: side, diff --git a/app/src/features/bots/useFollowBot.ts b/app/src/features/bots/useFollowBot.ts index 69e28a2f..e6c74bc3 100644 --- a/app/src/features/bots/useFollowBot.ts +++ b/app/src/features/bots/useFollowBot.ts @@ -1,7 +1,9 @@ // The panel beside the chat follows what the bot starts doing (spec 15.1), // so the owner sees it happen: its browser starting or a request for a // hand opens the browser (spec 21.8), and a screen it starts writing opens -// the design area (spec 22.5), whatever panel was open. Each opens once per +// the design area (spec 22.5), and the bot starting to use the owner's +// desktop after a pause opens the desktop panel (spec 24.9), whatever panel +// was open. Each opens once per // start; closing it leaves the button's dot. When the browser goes to rest // (spec 21.2) its panel closes, and it opens again once the bot wakes it. // The browser in the owner's hands is never taken away. The owner can turn this off in Settings @@ -12,9 +14,12 @@ import { useEffect, useRef } from "react"; import type { Bot } from "../../lib/protocol.gen"; import { prefs } from "../../shell/prefs"; -import { useApp } from "../../store/context"; +import { useApi, useApp } from "../../store/context"; -export type Followed = "browser" | "screens"; +export type Followed = "browser" | "screens" | "desktop"; + +/** A use of the desktop this long after the last one starts a new run. */ +const DESKTOP_PAUSE = 2 * 60_000; export function useFollowBot({ bot, @@ -41,6 +46,10 @@ export function useFollowBot({ // behind its button, a request for a hand does not. const before = useRef<{ browsing: boolean; asking: boolean } | null>(null); const drawn = useRef(new Set()); + const api = useApi(); + // The latest of what the desktop listener needs, without listening anew. + const now = useRef({ side, held, open }); + now.current = { side, held, open }; // biome-ignore lint/correctness/useExhaustiveDependencies: another bot starts afresh useEffect(() => { @@ -75,4 +84,26 @@ export function useFollowBot({ } open("screens"); }, [bot.id, writing]); + + useEffect(() => { + let last: number | null = null; + return api.subscribe((event) => { + if (event.name !== "desktop.changed" || event.params.botId !== bot.id) { + return; + } + const { at, window, stopped } = event.params; + if (at === null || !window || stopped || at === last) { + return; + } + const was = last; + last = at; + if (was !== null && at - was < DESKTOP_PAUSE) { + return; + } + const { side, held, open } = now.current; + if (side !== "desktop" && !held && prefs.followBot.get()) { + open("desktop"); + } + }); + }, [api, bot.id]); } diff --git a/app/src/features/browser/LiveView.tsx b/app/src/features/browser/LiveView.tsx index 04b20c2d..623d5eb0 100644 --- a/app/src/features/browser/LiveView.tsx +++ b/app/src/features/browser/LiveView.tsx @@ -1,17 +1,33 @@ // The bot's screen, live (spec 21.8): the newest picture of its active // tab, as large as fits the panel, with its cursor gliding to each point it -// acts on, a ring where it clicks, and a line about what it just did. +// acts on, a ring where it clicks, and a line about what it just did. The +// desktop panel shows the owner's window the same way (spec 24.9). import { type ReactNode, useEffect, useState } from "react"; import { useT } from "../../i18n"; -import type { Bot, BrowserAction, BrowserFrame } from "../../lib/protocol.gen"; +import type { Bot, BrowserAction } from "../../lib/protocol.gen"; import { BotCursor } from "../bots/BotCursor"; /** The page until a picture says its size. */ const PAGE = { width: 1280, height: 800 }; +/** A picture on the stage: a JPEG, base64, and its size. */ +export interface StageFrame { + data: string; + width: number; + height: number; +} + +/** What the bot did, at a point in the picture's pixels when it has one. */ +export interface StageAction { + kind: string; + x: number | null; + y: number | null; + at: number; +} + /** Where on the screen, in percent, an action happened. */ -function spot(action: BrowserAction | null, frame: BrowserFrame | null) { +function spot(action: StageAction | null, frame: StageFrame | null) { if (!action || action.x === null || action.y === null) { return null; } @@ -52,13 +68,19 @@ export function LiveView({ bot, frame, action, + label, + alt = "", dim = false, held = false, children, }: { bot: Bot; - frame: BrowserFrame | null; - action: BrowserAction | null; + frame: StageFrame | null; + action: StageAction | null; + /** What the stage is, for screen readers; the bot's browser by default. */ + label?: string; + /** The picture's own name, when it has one. */ + alt?: string; /** The browser closed: the last picture stays, faded. */ dim?: boolean; /** The owner has it (spec 21.10): the bot's cursor steps aside. */ @@ -81,7 +103,7 @@ export function LiveView({ return (

- {frame ? ( - {window.title} - ) : ( -

{t.waiting}

- )} +
+ {frame ? ( +
+ +
+ ) : ( +

{t.waiting}

+ )} +

diff --git a/app/src/features/desktop/panel.test.tsx b/app/src/features/desktop/panel.test.tsx index 75e86eed..ef181e96 100644 --- a/app/src/features/desktop/panel.test.tsx +++ b/app/src/features/desktop/panel.test.tsx @@ -34,7 +34,13 @@ describe("the desktop panel", () => { expect(fake.desktop.watching).toBe(scout.id); act(() => { - fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null }); + fake.desktop.use(scout.id, NOTES, { + kind: "click", + target: "Save", + option: null, + x: 0.5, + y: 0.25, + }); }); expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); expect(within(panel()).getByText('Clicked "Save"')).toBeDefined(); @@ -47,6 +53,13 @@ describe("the desktop panel", () => { expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA"); expect(within(panel()).getByText("Live")).toBeDefined(); + // The bot's cursor lands where it clicked, with a ring. + const stage = within(panel()).getByRole("figure", { name: "Notepad, as Scout sees it" }); + const cursor = stage.querySelector(".bot-cursor") as HTMLElement; + expect(cursor.style.left).toBe("50%"); + expect(cursor.style.top).toBe("25%"); + expect(stage.querySelector(".browser-ripple")).not.toBeNull(); + // Closed, nobody watches. fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); await waitFor(() => expect(fake.desktop.watching).toBeNull()); @@ -68,4 +81,31 @@ describe("the desktop panel", () => { await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false)); expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull(); }); + + test("opens by itself when the bot starts using the desktop, once per run", async () => { + const { fake, scout } = await openScout(); + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(); + act(() => { + fake.desktop.use(scout.id, NOTES); + }); + expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); + + // Closed, the next action of the same run leaves it closed. + fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); + await waitFor(() => + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(), + ); + act(() => { + fake.now += 5_000; + fake.desktop.use(scout.id, NOTES); + }); + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(); + + // After a pause, a new run opens it again. + act(() => { + fake.now += 3 * 60_000; + fake.desktop.use(scout.id, NOTES); + }); + expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); + }); }); diff --git a/app/src/i18n/en/desktop.ts b/app/src/i18n/en/desktop.ts index 1390f998..83e98cb2 100644 --- a/app/src/i18n/en/desktop.ts +++ b/app/src/i18n/en/desktop.ts @@ -20,6 +20,7 @@ export const desktop = { }, panel: { label: (bot: string) => `${bot}'s desktop`, + screen: (bot: string, app: string) => `${app}, as ${bot} sees it`, heading: "Desktop", live: "Live", stoppedBadge: "Stopped", diff --git a/app/src/i18n/es/desktop.ts b/app/src/i18n/es/desktop.ts index 310da7ba..41b4783c 100644 --- a/app/src/i18n/es/desktop.ts +++ b/app/src/i18n/es/desktop.ts @@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = { }, panel: { label: (bot) => `Escritorio de ${bot}`, + screen: (bot, app) => `${app}, como lo ve ${bot}`, heading: "Escritorio", live: "En vivo", stoppedBadge: "Detenido", diff --git a/app/src/i18n/pt-BR/desktop.ts b/app/src/i18n/pt-BR/desktop.ts index ecf0e4e8..2c66f00f 100644 --- a/app/src/i18n/pt-BR/desktop.ts +++ b/app/src/i18n/pt-BR/desktop.ts @@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = { }, panel: { label: (bot) => `Desktop de ${bot}`, + screen: (bot, app) => `${app}, como ${bot} vê`, heading: "Desktop", live: "Ao vivo", stoppedBadge: "Parado", diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index ad8ccfab..1ebc5d68 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -921,7 +921,13 @@ target: string, /** * The option chosen, for `select`. */ -option: string | null, }; +option: string | null, +/** + * Where in the window, as fractions of its width and height from its + * top left corner, when the action has a point: for the bot's cursor + * on its panel. + */ +x: number | null, y: number | null, }; /** * What a bot does on the owner's desktop, for its panel (spec 24.9): the diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 6759edcf..41fc4b15 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -173,7 +173,7 @@ pub enum DesktopActionKind { Press, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopAction { @@ -182,11 +182,16 @@ pub struct DesktopAction { pub target: String, /// The option chosen, for `select`. pub option: Option, + /// Where in the window, as fractions of its width and height from its + /// top left corner, when the action has a point: for the bot's cursor + /// on its panel. + pub x: Option, + pub y: Option, } /// What a bot does on the owner's desktop, for its panel (spec 24.9): the /// params of `desktop.changed`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopState { @@ -216,7 +221,7 @@ pub struct DesktopFrame { /// The result of `desktop.watch`: the bot's state and, when it uses a /// window, its picture now. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopView { diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs index 3c5d111a..aa97e149 100644 --- a/crates/botloftd/src/desktop/activity.rs +++ b/crates/botloftd/src/desktop/activity.rs @@ -108,6 +108,8 @@ mod tests { kind: botloft_core::protocol::DesktopActionKind::Click, target: "Save".to_owned(), option: None, + x: Some(0.5), + y: Some(0.25), }; let used = activity.used(&bot, &window, Some(click), 9); assert_eq!(activity.window(&bot), Some(7)); diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 5c2dcb4a..ddb1c936 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -246,6 +246,20 @@ pub fn real_press(id: u64, keys: &keys::Keys) -> Result<(), DesktopError> { } } +/// Where window `id`'s frame is on the screen, in real pixels: left, top, +/// width and height. +pub fn frame(id: u64) -> Result<[i32; 4], DesktopError> { + #[cfg(windows)] + { + windows::frame(id) + } + #[cfg(not(windows))] + { + let _ = id; + Err(DesktopError::Unavailable) + } +} + /// Shows the notice that a bot uses the real mouse and keyboard: `text` /// in a pill, a border in `color` around the monitor of window `over` /// (spec 24.7). diff --git a/crates/botloftd/src/platform/desktop/windows/capture.rs b/crates/botloftd/src/platform/desktop/windows/capture.rs index 2a2636d2..ce880a47 100644 --- a/crates/botloftd/src/platform/desktop/windows/capture.rs +++ b/crates/botloftd/src/platform/desktop/windows/capture.rs @@ -70,6 +70,32 @@ fn system(what: &str) -> DesktopError { DesktopError::System(what.to_owned()) } +/// Where window `id`'s frame is on the screen, in real pixels, as the +/// controls' places are: left, top, width and height. +pub fn frame(id: u64) -> Result<[i32; 4], DesktopError> { + let hwnd = handle(id); + let _real = RealPixels::new(); + let mut frame = RECT::default(); + // SAFETY: `frame` is a valid out-pointer. + unsafe { GetWindowRect(hwnd, &mut frame) }.map_err(|_| DesktopError::Gone)?; + // SAFETY: `frame` is a RECT of the size given. Without DWM the window + // rectangle is the frame. + let _ = unsafe { + DwmGetWindowAttribute( + hwnd, + DWMWA_EXTENDED_FRAME_BOUNDS, + std::ptr::from_mut(&mut frame).cast::(), + size_of::() as u32, + ) + }; + Ok([ + frame.left, + frame.top, + frame.right - frame.left, + frame.bottom - frame.top, + ]) +} + /// The window `id`'s pixels as RGB, its width and height, and the display /// scale it is on (1.5 at 150%). pub fn capture(id: u64) -> Result<(Vec, u32, u32, f64), DesktopError> { diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 0b50aa43..42e981b1 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -35,7 +35,7 @@ use windows::Win32::UI::WindowsAndMessaging::{ use windows::core::{BOOL, w}; pub use act::act; -pub use capture::capture; +pub use capture::{capture, frame}; pub use input::owner_idle; pub use notice::{hide as notice_hide, show as notice_show}; pub use real::{click as real_click, press as real_press, type_text as real_type}; diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index bfdd1682..0cc23efd 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -57,9 +57,24 @@ fn shown(action: &Action, control: &Control) -> DesktopAction { kind, target: control.name.clone(), option, + x: None, + y: None, } } +/// The middle of a control's place, as fractions of the window's `frame`; +/// nothing when it has no place or lies outside the window. +fn point(rect: Option<[i32; 4]>, frame: [i32; 4]) -> Option<(f64, f64)> { + let [left, top, width, height] = rect?; + let [frame_left, frame_top, frame_width, frame_height] = frame; + if frame_width <= 0 || frame_height <= 0 { + return None; + } + let x = f64::from(left + width / 2 - frame_left) / f64::from(frame_width); + let y = f64::from(top + height / 2 - frame_top) / f64::from(frame_height); + ((0.0..=1.0).contains(&x) && (0.0..=1.0).contains(&y)).then_some((x, y)) +} + /// The window read again after an action, with `said` first: kept as the /// bot's reading and shown on its panel. pub(super) async fn answered( @@ -170,6 +185,24 @@ pub(super) async fn act( enter(daemon, bot, &window).await?; said.push_str(" Pressed Enter."); } - let action = shown(&ask.action, &control); + let mut action = shown(&ask.action, &control); + let id = window.id; + if let Ok(frame) = blocking(move || platform::frame(id)).await { + (action.x, action.y) = point(control.rect, frame).unzip(); + } answered(daemon, bot, window, said, action).await } + +#[cfg(test)] +mod tests { + use super::point; + + #[test] + fn a_controls_point_is_its_middle_within_the_window() { + let frame = [100, 50, 400, 200]; + assert_eq!(point(Some([200, 100, 20, 10]), frame), Some((0.275, 0.275))); + assert_eq!(point(Some([600, 100, 20, 10]), frame), None, "outside"); + assert_eq!(point(None, frame), None); + assert_eq!(point(Some([200, 100, 20, 10]), [0, 0, 0, 0]), None); + } +} diff --git a/crates/botloftd/src/tools/desktop_real.rs b/crates/botloftd/src/tools/desktop_real.rs index b2cf5f93..64ba639a 100644 --- a/crates/botloftd/src/tools/desktop_real.rs +++ b/crates/botloftd/src/tools/desktop_real.rs @@ -73,6 +73,8 @@ pub(super) async fn press( kind: DesktopActionKind::Press, target: String::new(), option: Some(text.trim().to_owned()), + x: None, + y: None, }) } @@ -99,16 +101,16 @@ pub(super) async fn click_at( "That point is outside the picture, which is {width} by {height}." )); } - let spot = platform::Spot::Window { - x: x / f64::from(width), - y: y / f64::from(height), - }; + let (x, y) = (x / f64::from(width), y / f64::from(height)); + let spot = platform::Spot::Window { x, y }; let id = window.id; with_real_hands(daemon, bot, id, move || platform::real_click(id, spot)).await?; Ok(DesktopAction { kind: DesktopActionKind::Click, target: String::new(), option: None, + x: Some(x), + y: Some(y), }) } diff --git a/crates/botloftd/tests/desktop_panel.rs b/crates/botloftd/tests/desktop_panel.rs index 5d6110a0..283e869c 100644 --- a/crates/botloftd/tests/desktop_panel.rs +++ b/crates/botloftd/tests/desktop_panel.rs @@ -6,7 +6,7 @@ mod common; use common::browsing::{call, pending_approval}; -use common::desktop::{TITLE, TestWindow, reading, setup}; +use common::desktop::{TITLE, TestWindow, reading, ref_in, setup}; use serde_json::json; #[tokio::test(flavor = "multi_thread")] @@ -30,7 +30,7 @@ async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { let asked = pending_approval(&mut s.app).await; let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); s.app.call("approvals.answer", allow).await.expect("answer"); - looking.await.expect("task").expect("reading"); + let text = looking.await.expect("task").expect("reading"); // The panel hears which window, and its picture follows. let changed = s.app.notification("desktop.changed").await; @@ -40,6 +40,25 @@ async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { assert_eq!(frame["botId"], s.bot["id"]); assert!(frame["data"].as_str().is_some_and(|data| data.len() > 100)); + // A click says where it was in the window, for the bot's cursor: the + // Save button is near the top left. + let save = ref_in(&text, r#"button "Save""#); + let click = json!({ "ref": save, "why": "To save the visit" }); + let clicking = call(&s.mcp, "desktop_click", click); + let asked = pending_approval(&mut s.app).await; + let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); + s.app.call("approvals.answer", allow).await.expect("answer"); + clicking.await.expect("task").expect("clicked"); + let clicked = loop { + let changed = s.app.notification("desktop.changed").await; + if changed["action"]["kind"] == "click" { + break changed; + } + }; + let (x, y) = (&clicked["action"]["x"], &clicked["action"]["y"]); + assert!(x.as_f64().is_some_and(|x| x > 0.0 && x < 0.5), "{clicked}"); + assert!(y.as_f64().is_some_and(|y| y > 0.0 && y < 0.5), "{clicked}"); + s.app .call("desktop.unwatch", serde_json::Value::Null) .await diff --git a/docs/spec.md b/docs/spec.md index c79cd1b7..6ad8e064 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1764,8 +1764,8 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. -- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. Sobre a foto, o cursor do bot desliza até o ponto de cada ação e um anel marca o clique, como no navegador (21.8). O painel abre sozinho quando o bot começa a usar o desktop, isto é, no primeiro `desktop.changed` com janela depois de 2 minutos sem nenhum, sobre o painel que estiver aberto, menos com o navegador nas mãos do dono, e só se o dono não desligou "seguir o bot" (15.1); fechado no meio, as ações seguintes da mesma vez não o abrem de novo. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle, a opção escolhida e o ponto, `x` e `y` em frações da moldura da janela a partir do canto de cima à esquerda: o centro do controle pela acessibilidade, ou o ponto da foto num clique de `desktop_click_at`, sem ponto nas teclas ou com o controle fora da janela; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. - **Parar tudo:** o atalho global `Ctrl+Alt+Esc`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Esc está em todo teclado, até nos compactos (60% e 70% não têm End), e o Windows não usa Ctrl+Alt+Esc (os dele são Ctrl+Shift+Esc, Alt+Esc e Ctrl+Esc); um bot não o aperta, porque Alt+Esc está entre as teclas do sistema (24.7). Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele.