diff --git a/app/src/dev/seedDesktop.ts b/app/src/dev/seedDesktop.ts index 0ce7e2a5..3674baa2 100644 --- a/app/src/dev/seedDesktop.ts +++ b/app/src/dev/seedDesktop.ts @@ -34,7 +34,13 @@ function drawNotepad(): string { export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) { fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad"); - fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null }); + fake.desktop.use(bots.scout, NOTES, { + kind: "click", + target: "Save", + option: null, + x: 0.1, + y: 0.12, + }); fake.desktop.paint(bots.scout, drawNotepad(), 760, 480); fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", { summary: "To save the list", diff --git a/app/src/features/bots/BotView.tsx b/app/src/features/bots/BotView.tsx index 83559fdf..44e52f97 100644 --- a/app/src/features/bots/BotView.tsx +++ b/app/src/features/bots/BotView.tsx @@ -121,15 +121,6 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { setTakeBrowser((count) => count + 1); } }); - // The panel follows what the bot starts doing (spec 15.1), over the one - // that came back too. - useFollowBot({ - bot, - side, - writing: screens.writing, - open: (panel) => (panel === "browser" ? showBrowser() : showScreen(null)), - close: (panel) => side === panel && setSide(null), - }); const showTerminal = useStable(() => { if (filesOpen) { seen(); @@ -142,6 +133,23 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { } setSide("desktop"); }); + // The panel follows what the bot starts doing (spec 15.1), over the one + // that came back too. + useFollowBot({ + bot, + side, + writing: screens.writing, + open: (panel) => { + if (panel === "browser") { + showBrowser(); + } else if (panel === "desktop") { + showDesktop(); + } else { + showScreen(null); + } + }, + close: (panel) => side === panel && setSide(null), + }); // The dock of the bot's computer: browser, terminal and files. const dock = { open: side, diff --git a/app/src/features/bots/useFollowBot.ts b/app/src/features/bots/useFollowBot.ts index 69e28a2f..e6c74bc3 100644 --- a/app/src/features/bots/useFollowBot.ts +++ b/app/src/features/bots/useFollowBot.ts @@ -1,7 +1,9 @@ // The panel beside the chat follows what the bot starts doing (spec 15.1), // so the owner sees it happen: its browser starting or a request for a // hand opens the browser (spec 21.8), and a screen it starts writing opens -// the design area (spec 22.5), whatever panel was open. Each opens once per +// the design area (spec 22.5), and the bot starting to use the owner's +// desktop after a pause opens the desktop panel (spec 24.9), whatever panel +// was open. Each opens once per // start; closing it leaves the button's dot. When the browser goes to rest // (spec 21.2) its panel closes, and it opens again once the bot wakes it. // The browser in the owner's hands is never taken away. The owner can turn this off in Settings @@ -12,9 +14,12 @@ import { useEffect, useRef } from "react"; import type { Bot } from "../../lib/protocol.gen"; import { prefs } from "../../shell/prefs"; -import { useApp } from "../../store/context"; +import { useApi, useApp } from "../../store/context"; -export type Followed = "browser" | "screens"; +export type Followed = "browser" | "screens" | "desktop"; + +/** A use of the desktop this long after the last one starts a new run. */ +const DESKTOP_PAUSE = 2 * 60_000; export function useFollowBot({ bot, @@ -41,6 +46,10 @@ export function useFollowBot({ // behind its button, a request for a hand does not. const before = useRef<{ browsing: boolean; asking: boolean } | null>(null); const drawn = useRef(new Set()); + const api = useApi(); + // The latest of what the desktop listener needs, without listening anew. + const now = useRef({ side, held, open }); + now.current = { side, held, open }; // biome-ignore lint/correctness/useExhaustiveDependencies: another bot starts afresh useEffect(() => { @@ -75,4 +84,26 @@ export function useFollowBot({ } open("screens"); }, [bot.id, writing]); + + useEffect(() => { + let last: number | null = null; + return api.subscribe((event) => { + if (event.name !== "desktop.changed" || event.params.botId !== bot.id) { + return; + } + const { at, window, stopped } = event.params; + if (at === null || !window || stopped || at === last) { + return; + } + const was = last; + last = at; + if (was !== null && at - was < DESKTOP_PAUSE) { + return; + } + const { side, held, open } = now.current; + if (side !== "desktop" && !held && prefs.followBot.get()) { + open("desktop"); + } + }); + }, [api, bot.id]); } diff --git a/app/src/features/browser/LiveView.tsx b/app/src/features/browser/LiveView.tsx index 04b20c2d..623d5eb0 100644 --- a/app/src/features/browser/LiveView.tsx +++ b/app/src/features/browser/LiveView.tsx @@ -1,17 +1,33 @@ // The bot's screen, live (spec 21.8): the newest picture of its active // tab, as large as fits the panel, with its cursor gliding to each point it -// acts on, a ring where it clicks, and a line about what it just did. +// acts on, a ring where it clicks, and a line about what it just did. The +// desktop panel shows the owner's window the same way (spec 24.9). import { type ReactNode, useEffect, useState } from "react"; import { useT } from "../../i18n"; -import type { Bot, BrowserAction, BrowserFrame } from "../../lib/protocol.gen"; +import type { Bot, BrowserAction } from "../../lib/protocol.gen"; import { BotCursor } from "../bots/BotCursor"; /** The page until a picture says its size. */ const PAGE = { width: 1280, height: 800 }; +/** A picture on the stage: a JPEG, base64, and its size. */ +export interface StageFrame { + data: string; + width: number; + height: number; +} + +/** What the bot did, at a point in the picture's pixels when it has one. */ +export interface StageAction { + kind: string; + x: number | null; + y: number | null; + at: number; +} + /** Where on the screen, in percent, an action happened. */ -function spot(action: BrowserAction | null, frame: BrowserFrame | null) { +function spot(action: StageAction | null, frame: StageFrame | null) { if (!action || action.x === null || action.y === null) { return null; } @@ -52,13 +68,19 @@ export function LiveView({ bot, frame, action, + label, + alt = "", dim = false, held = false, children, }: { bot: Bot; - frame: BrowserFrame | null; - action: BrowserAction | null; + frame: StageFrame | null; + action: StageAction | null; + /** What the stage is, for screen readers; the bot's browser by default. */ + label?: string; + /** The picture's own name, when it has one. */ + alt?: string; /** The browser closed: the last picture stays, faded. */ dim?: boolean; /** The owner has it (spec 21.10): the bot's cursor steps aside. */ @@ -81,7 +103,7 @@ export function LiveView({ return (

- {frame ? ( - {window.title} - ) : ( -

{t.waiting}

- )} +
+ {frame ? ( +
+ +
+ ) : ( +

{t.waiting}

+ )} +

diff --git a/app/src/features/desktop/panel.test.tsx b/app/src/features/desktop/panel.test.tsx index 75e86eed..ef181e96 100644 --- a/app/src/features/desktop/panel.test.tsx +++ b/app/src/features/desktop/panel.test.tsx @@ -34,7 +34,13 @@ describe("the desktop panel", () => { expect(fake.desktop.watching).toBe(scout.id); act(() => { - fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null }); + fake.desktop.use(scout.id, NOTES, { + kind: "click", + target: "Save", + option: null, + x: 0.5, + y: 0.25, + }); }); expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); expect(within(panel()).getByText('Clicked "Save"')).toBeDefined(); @@ -47,6 +53,13 @@ describe("the desktop panel", () => { expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA"); expect(within(panel()).getByText("Live")).toBeDefined(); + // The bot's cursor lands where it clicked, with a ring. + const stage = within(panel()).getByRole("figure", { name: "Notepad, as Scout sees it" }); + const cursor = stage.querySelector(".bot-cursor") as HTMLElement; + expect(cursor.style.left).toBe("50%"); + expect(cursor.style.top).toBe("25%"); + expect(stage.querySelector(".browser-ripple")).not.toBeNull(); + // Closed, nobody watches. fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); await waitFor(() => expect(fake.desktop.watching).toBeNull()); @@ -68,4 +81,31 @@ describe("the desktop panel", () => { await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false)); expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull(); }); + + test("opens by itself when the bot starts using the desktop, once per run", async () => { + const { fake, scout } = await openScout(); + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(); + act(() => { + fake.desktop.use(scout.id, NOTES); + }); + expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); + + // Closed, the next action of the same run leaves it closed. + fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); + await waitFor(() => + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(), + ); + act(() => { + fake.now += 5_000; + fake.desktop.use(scout.id, NOTES); + }); + expect(screen.queryByRole("complementary", { name: "Scout's desktop" })).toBeNull(); + + // After a pause, a new run opens it again. + act(() => { + fake.now += 3 * 60_000; + fake.desktop.use(scout.id, NOTES); + }); + expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); + }); }); diff --git a/app/src/i18n/en/desktop.ts b/app/src/i18n/en/desktop.ts index 1390f998..83e98cb2 100644 --- a/app/src/i18n/en/desktop.ts +++ b/app/src/i18n/en/desktop.ts @@ -20,6 +20,7 @@ export const desktop = { }, panel: { label: (bot: string) => `${bot}'s desktop`, + screen: (bot: string, app: string) => `${app}, as ${bot} sees it`, heading: "Desktop", live: "Live", stoppedBadge: "Stopped", diff --git a/app/src/i18n/es/desktop.ts b/app/src/i18n/es/desktop.ts index 310da7ba..41b4783c 100644 --- a/app/src/i18n/es/desktop.ts +++ b/app/src/i18n/es/desktop.ts @@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = { }, panel: { label: (bot) => `Escritorio de ${bot}`, + screen: (bot, app) => `${app}, como lo ve ${bot}`, heading: "Escritorio", live: "En vivo", stoppedBadge: "Detenido", diff --git a/app/src/i18n/pt-BR/desktop.ts b/app/src/i18n/pt-BR/desktop.ts index ecf0e4e8..2c66f00f 100644 --- a/app/src/i18n/pt-BR/desktop.ts +++ b/app/src/i18n/pt-BR/desktop.ts @@ -19,6 +19,7 @@ export const desktop: Messages["desktop"] = { }, panel: { label: (bot) => `Desktop de ${bot}`, + screen: (bot, app) => `${app}, como ${bot} vê`, heading: "Desktop", live: "Ao vivo", stoppedBadge: "Parado", diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index ad8ccfab..1ebc5d68 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -921,7 +921,13 @@ target: string, /** * The option chosen, for `select`. */ -option: string | null, }; +option: string | null, +/** + * Where in the window, as fractions of its width and height from its + * top left corner, when the action has a point: for the bot's cursor + * on its panel. + */ +x: number | null, y: number | null, }; /** * What a bot does on the owner's desktop, for its panel (spec 24.9): the diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 6759edcf..41fc4b15 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -173,7 +173,7 @@ pub enum DesktopActionKind { Press, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopAction { @@ -182,11 +182,16 @@ pub struct DesktopAction { pub target: String, /// The option chosen, for `select`. pub option: Option, + /// Where in the window, as fractions of its width and height from its + /// top left corner, when the action has a point: for the bot's cursor + /// on its panel. + pub x: Option, + pub y: Option, } /// What a bot does on the owner's desktop, for its panel (spec 24.9): the /// params of `desktop.changed`. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopState { @@ -216,7 +221,7 @@ pub struct DesktopFrame { /// The result of `desktop.watch`: the bot's state and, when it uses a /// window, its picture now. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[cfg_attr(test, derive(ts_rs::TS))] pub struct DesktopView { diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs index 3c5d111a..aa97e149 100644 --- a/crates/botloftd/src/desktop/activity.rs +++ b/crates/botloftd/src/desktop/activity.rs @@ -108,6 +108,8 @@ mod tests { kind: botloft_core::protocol::DesktopActionKind::Click, target: "Save".to_owned(), option: None, + x: Some(0.5), + y: Some(0.25), }; let used = activity.used(&bot, &window, Some(click), 9); assert_eq!(activity.window(&bot), Some(7)); diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 5c2dcb4a..ddb1c936 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -246,6 +246,20 @@ pub fn real_press(id: u64, keys: &keys::Keys) -> Result<(), DesktopError> { } } +/// Where window `id`'s frame is on the screen, in real pixels: left, top, +/// width and height. +pub fn frame(id: u64) -> Result<[i32; 4], DesktopError> { + #[cfg(windows)] + { + windows::frame(id) + } + #[cfg(not(windows))] + { + let _ = id; + Err(DesktopError::Unavailable) + } +} + /// Shows the notice that a bot uses the real mouse and keyboard: `text` /// in a pill, a border in `color` around the monitor of window `over` /// (spec 24.7). diff --git a/crates/botloftd/src/platform/desktop/windows/capture.rs b/crates/botloftd/src/platform/desktop/windows/capture.rs index 2a2636d2..ce880a47 100644 --- a/crates/botloftd/src/platform/desktop/windows/capture.rs +++ b/crates/botloftd/src/platform/desktop/windows/capture.rs @@ -70,6 +70,32 @@ fn system(what: &str) -> DesktopError { DesktopError::System(what.to_owned()) } +/// Where window `id`'s frame is on the screen, in real pixels, as the +/// controls' places are: left, top, width and height. +pub fn frame(id: u64) -> Result<[i32; 4], DesktopError> { + let hwnd = handle(id); + let _real = RealPixels::new(); + let mut frame = RECT::default(); + // SAFETY: `frame` is a valid out-pointer. + unsafe { GetWindowRect(hwnd, &mut frame) }.map_err(|_| DesktopError::Gone)?; + // SAFETY: `frame` is a RECT of the size given. Without DWM the window + // rectangle is the frame. + let _ = unsafe { + DwmGetWindowAttribute( + hwnd, + DWMWA_EXTENDED_FRAME_BOUNDS, + std::ptr::from_mut(&mut frame).cast::(), + size_of::() as u32, + ) + }; + Ok([ + frame.left, + frame.top, + frame.right - frame.left, + frame.bottom - frame.top, + ]) +} + /// The window `id`'s pixels as RGB, its width and height, and the display /// scale it is on (1.5 at 150%). pub fn capture(id: u64) -> Result<(Vec, u32, u32, f64), DesktopError> { diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 0b50aa43..42e981b1 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -35,7 +35,7 @@ use windows::Win32::UI::WindowsAndMessaging::{ use windows::core::{BOOL, w}; pub use act::act; -pub use capture::capture; +pub use capture::{capture, frame}; pub use input::owner_idle; pub use notice::{hide as notice_hide, show as notice_show}; pub use real::{click as real_click, press as real_press, type_text as real_type}; diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index bfdd1682..0cc23efd 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -57,9 +57,24 @@ fn shown(action: &Action, control: &Control) -> DesktopAction { kind, target: control.name.clone(), option, + x: None, + y: None, } } +/// The middle of a control's place, as fractions of the window's `frame`; +/// nothing when it has no place or lies outside the window. +fn point(rect: Option<[i32; 4]>, frame: [i32; 4]) -> Option<(f64, f64)> { + let [left, top, width, height] = rect?; + let [frame_left, frame_top, frame_width, frame_height] = frame; + if frame_width <= 0 || frame_height <= 0 { + return None; + } + let x = f64::from(left + width / 2 - frame_left) / f64::from(frame_width); + let y = f64::from(top + height / 2 - frame_top) / f64::from(frame_height); + ((0.0..=1.0).contains(&x) && (0.0..=1.0).contains(&y)).then_some((x, y)) +} + /// The window read again after an action, with `said` first: kept as the /// bot's reading and shown on its panel. pub(super) async fn answered( @@ -170,6 +185,24 @@ pub(super) async fn act( enter(daemon, bot, &window).await?; said.push_str(" Pressed Enter."); } - let action = shown(&ask.action, &control); + let mut action = shown(&ask.action, &control); + let id = window.id; + if let Ok(frame) = blocking(move || platform::frame(id)).await { + (action.x, action.y) = point(control.rect, frame).unzip(); + } answered(daemon, bot, window, said, action).await } + +#[cfg(test)] +mod tests { + use super::point; + + #[test] + fn a_controls_point_is_its_middle_within_the_window() { + let frame = [100, 50, 400, 200]; + assert_eq!(point(Some([200, 100, 20, 10]), frame), Some((0.275, 0.275))); + assert_eq!(point(Some([600, 100, 20, 10]), frame), None, "outside"); + assert_eq!(point(None, frame), None); + assert_eq!(point(Some([200, 100, 20, 10]), [0, 0, 0, 0]), None); + } +} diff --git a/crates/botloftd/src/tools/desktop_real.rs b/crates/botloftd/src/tools/desktop_real.rs index b2cf5f93..64ba639a 100644 --- a/crates/botloftd/src/tools/desktop_real.rs +++ b/crates/botloftd/src/tools/desktop_real.rs @@ -73,6 +73,8 @@ pub(super) async fn press( kind: DesktopActionKind::Press, target: String::new(), option: Some(text.trim().to_owned()), + x: None, + y: None, }) } @@ -99,16 +101,16 @@ pub(super) async fn click_at( "That point is outside the picture, which is {width} by {height}." )); } - let spot = platform::Spot::Window { - x: x / f64::from(width), - y: y / f64::from(height), - }; + let (x, y) = (x / f64::from(width), y / f64::from(height)); + let spot = platform::Spot::Window { x, y }; let id = window.id; with_real_hands(daemon, bot, id, move || platform::real_click(id, spot)).await?; Ok(DesktopAction { kind: DesktopActionKind::Click, target: String::new(), option: None, + x: Some(x), + y: Some(y), }) } diff --git a/crates/botloftd/tests/desktop_panel.rs b/crates/botloftd/tests/desktop_panel.rs index 5d6110a0..283e869c 100644 --- a/crates/botloftd/tests/desktop_panel.rs +++ b/crates/botloftd/tests/desktop_panel.rs @@ -6,7 +6,7 @@ mod common; use common::browsing::{call, pending_approval}; -use common::desktop::{TITLE, TestWindow, reading, setup}; +use common::desktop::{TITLE, TestWindow, reading, ref_in, setup}; use serde_json::json; #[tokio::test(flavor = "multi_thread")] @@ -30,7 +30,7 @@ async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { let asked = pending_approval(&mut s.app).await; let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); s.app.call("approvals.answer", allow).await.expect("answer"); - looking.await.expect("task").expect("reading"); + let text = looking.await.expect("task").expect("reading"); // The panel hears which window, and its picture follows. let changed = s.app.notification("desktop.changed").await; @@ -40,6 +40,25 @@ async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { assert_eq!(frame["botId"], s.bot["id"]); assert!(frame["data"].as_str().is_some_and(|data| data.len() > 100)); + // A click says where it was in the window, for the bot's cursor: the + // Save button is near the top left. + let save = ref_in(&text, r#"button "Save""#); + let click = json!({ "ref": save, "why": "To save the visit" }); + let clicking = call(&s.mcp, "desktop_click", click); + let asked = pending_approval(&mut s.app).await; + let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); + s.app.call("approvals.answer", allow).await.expect("answer"); + clicking.await.expect("task").expect("clicked"); + let clicked = loop { + let changed = s.app.notification("desktop.changed").await; + if changed["action"]["kind"] == "click" { + break changed; + } + }; + let (x, y) = (&clicked["action"]["x"], &clicked["action"]["y"]); + assert!(x.as_f64().is_some_and(|x| x > 0.0 && x < 0.5), "{clicked}"); + assert!(y.as_f64().is_some_and(|y| y > 0.0 && y < 0.5), "{clicked}"); + s.app .call("desktop.unwatch", serde_json::Value::Null) .await diff --git a/docs/spec.md b/docs/spec.md index c79cd1b7..6ad8e064 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1764,8 +1764,8 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. -- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. Sobre a foto, o cursor do bot desliza até o ponto de cada ação e um anel marca o clique, como no navegador (21.8). O painel abre sozinho quando o bot começa a usar o desktop, isto é, no primeiro `desktop.changed` com janela depois de 2 minutos sem nenhum, sobre o painel que estiver aberto, menos com o navegador nas mãos do dono, e só se o dono não desligou "seguir o bot" (15.1); fechado no meio, as ações seguintes da mesma vez não o abrem de novo. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle, a opção escolhida e o ponto, `x` e `y` em frações da moldura da janela a partir do canto de cima à esquerda: o centro do controle pela acessibilidade, ou o ponto da foto num clique de `desktop_click_at`, sem ponto nas teclas ou com o controle fora da janela; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. - **Parar tudo:** o atalho global `Ctrl+Alt+Esc`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Esc está em todo teclado, até nos compactos (60% e 70% não têm End), e o Windows não usa Ctrl+Alt+Esc (os dele são Ctrl+Shift+Esc, Alt+Esc e Ctrl+Esc); um bot não o aperta, porque Alt+Esc está entre as teclas do sistema (24.7). Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele.