From 810eceedbc860ab26e77273b544913eb249783a5 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:24:40 -0300 Subject: [PATCH 1/3] feat(desktop): what each bot does on the desktop, live, and stopping it The daemon part of D3 (spec 24.9): - Each bot's DesktopState, in memory: the window it read or acted in last, its last action as the tool said it, when, and whether the owner stopped it; every reading, picture and action updates it and sends desktop.changed. - desktop.watch and desktop.unwatch: while a connection watches a bot, its window is pictured four times a second and desktop.frame goes out only when the picture changed, only to that connection, newest first, like the browser's frames. The picturing stops with the last watcher. - desktop.stop and desktop.resume: a stopped bot's desktop tools refuse, again after waiting for the owner or for its turn, until the owner lets it go on; an action under way ends within its 5 seconds. - Ctrl+Alt+End, registered by the daemon on a thread with its own queue, stops every bot on the desktop, app open or not; when another program holds it, the daemon logs it and goes on. The window and setup of the desktop tests move to tests/common, and the event-to-notification mapping out of rpc/mod.rs, which had reached 300 lines. FakeBotloft plays the new methods. --- app/src/lib/fakeDesktop.ts | 66 ++++++- app/src/lib/protocol.gen.ts | 59 ++++++ crates/botloft-core/src/protocol/desktop.rs | 60 +++++++ .../src/protocol/export_desktop.rs | 19 ++ crates/botloft-core/src/protocol/mod.rs | 7 + crates/botloftd/src/desktop/activity.rs | 128 +++++++++++++ .../src/{desktop.rs => desktop/mod.rs} | 30 +++- crates/botloftd/src/desktop/screen.rs | 100 +++++++++++ crates/botloftd/src/main.rs | 8 +- crates/botloftd/src/platform/desktop/mod.rs | 14 ++ .../src/platform/desktop/windows/mod.rs | 2 + .../src/platform/desktop/windows/stop_key.rs | 52 ++++++ crates/botloftd/src/rpc/desktop_watch.rs | 50 ++++++ crates/botloftd/src/rpc/dispatch.rs | 2 + crates/botloftd/src/rpc/mod.rs | 48 ++--- crates/botloftd/src/rpc/notify.rs | 34 ++++ crates/botloftd/src/service/desktop.rs | 46 ++++- crates/botloftd/src/state.rs | 4 +- crates/botloftd/src/tools/desktop.rs | 22 ++- crates/botloftd/src/tools/desktop_act.rs | 7 + crates/botloftd/tests/common/desktop.rs | 169 ++++++++++++++++++ crates/botloftd/tests/common/mod.rs | 2 + crates/botloftd/tests/desktop.rs | 165 +---------------- crates/botloftd/tests/desktop_panel.rs | 83 +++++++++ docs/spec.md | 11 +- 25 files changed, 978 insertions(+), 210 deletions(-) create mode 100644 crates/botloftd/src/desktop/activity.rs rename crates/botloftd/src/{desktop.rs => desktop/mod.rs} (83%) create mode 100644 crates/botloftd/src/desktop/screen.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/stop_key.rs create mode 100644 crates/botloftd/src/rpc/desktop_watch.rs create mode 100644 crates/botloftd/src/rpc/notify.rs create mode 100644 crates/botloftd/tests/common/desktop.rs create mode 100644 crates/botloftd/tests/desktop_panel.rs diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index 4483ea44..fabd6cbc 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -1,16 +1,28 @@ -// The fake daemon's desktop grants (spec 24.2, 24.10): what each bot may -// see and do on the owner's desktop, and `desktop.*`. The bots' desktop -// tools are not played: grants come from `grant`, as the real daemon makes -// them when the owner allows a request. +// The fake daemon's desktop (spec 24): what each bot may see and do on the +// owner's desktop (spec 24.2, 24.10), what it is doing there for its panel, +// with pictures while the panel watches, and stopping it (spec 24.9). The +// bots' desktop tools are not played: grants come from `grant` and use +// from `use`, as the real daemon makes them. import type { FakeBotloft, Handlers } from "./fake"; import { notFound } from "./fakeRules"; -import type { BotId, DesktopGrant, DesktopLevel } from "./protocol.gen"; +import type { + BotId, + DesktopFrame, + DesktopGrant, + DesktopLevel, + DesktopState, + DesktopWindow, +} from "./protocol.gen"; type DesktopMethods = Extract; export class FakeDesktop { readonly grants: DesktopGrant[] = []; + private readonly states = new Map(); + private readonly frames = new Map(); + /** The bot whose panel the app watches, if any. */ + watching: BotId | null = null; constructor(private readonly fake: FakeBotloft) {} @@ -48,6 +60,33 @@ export class FakeDesktop { this.fake.emit({ name: "bot.desktop", params: { botId, grants: this.of(botId) } }); } + state(botId: BotId): DesktopState { + return ( + this.states.get(botId) ?? { botId, window: null, action: null, at: null, stopped: false } + ); + } + + private set(botId: BotId, change: Partial): DesktopState { + const state = { ...this.state(botId), ...change }; + this.states.set(botId, state); + this.fake.emit({ name: "desktop.changed", params: state }); + return state; + } + + /** The bot read or acted in `window`; `action` says what it did. */ + use(botId: BotId, window: DesktopWindow, action: string | null = null): DesktopState { + return this.set(botId, { window, action, at: this.fake.now }); + } + + /** A new picture of the bot's window: sent while its panel watches. */ + paint(botId: BotId, data: string, width = 800, height = 600): void { + const frame = { botId, data, width, height }; + this.frames.set(botId, frame); + if (this.watching === botId) { + this.fake.emit({ name: "desktop.frame", params: frame }); + } + } + handlers(): Pick { return { "desktop.grants": ({ botId }) => { @@ -64,6 +103,23 @@ export class FakeDesktop { this.changed(grant.botId); return { botId: grant.botId, grants: this.of(grant.botId) }; }, + "desktop.watch": ({ botId }) => { + this.fake.bot(botId); + this.watching = botId; + return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; + }, + "desktop.unwatch": () => { + this.watching = null; + return null; + }, + "desktop.stop": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: true }); + }, + "desktop.resume": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: false }); + }, }; } } diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 739d0b57..a5c29847 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -869,6 +869,59 @@ export type BotDesktop = { botId: BotId, */ grants: Array, }; +/** + * The window a bot is using on the owner's desktop. + */ +export type DesktopWindow = { id: number, title: string, +/** + * The app's name for people. + */ +app: string, }; + +/** + * What a bot does on the owner's desktop, for its panel (spec 24.9): the + * params of `desktop.changed`. + */ +export type DesktopState = { botId: BotId, +/** + * The window it read or acted in last. + */ +window: DesktopWindow | null, +/** + * What it did last, in English, as its tool said ("Clicked button + * \"Save\"."); `null` after only reading. + */ +action: string | null, +/** + * When it last read or acted, Unix ms. + */ +at: number | null, +/** + * The owner stopped it (spec 24.9): its desktop tools refuse until + * they let it go on. + */ +stopped: boolean, }; + +/** + * A picture of the window a bot is using, for whoever watches its panel. + */ +export type DesktopFrame = { botId: BotId, +/** + * JPEG, base64. + */ +data: string, width: number, height: number, }; + +/** + * The result of `desktop.watch`: the bot's state and, when it uses a + * window, its picture now. + */ +export type DesktopView = { state: DesktopState, frame: DesktopFrame | null, }; + +/** + * `desktop.watch`, `desktop.stop` and `desktop.resume`. + */ +export type DesktopBotParams = { botId: BotId, }; + export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, /** * What the bot says the command is for, as in `ToolItem`. The bot @@ -1551,6 +1604,10 @@ export interface RpcMethods { "rules.delete": { params: RuleIdParams; result: BotRules }; "desktop.grants": { params: DesktopGrantsParams; result: Array }; "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop }; + "desktop.watch": { params: DesktopBotParams; result: DesktopView }; + "desktop.unwatch": { params: undefined; result: null }; + "desktop.stop": { params: DesktopBotParams; result: DesktopState }; + "desktop.resume": { params: DesktopBotParams; result: DesktopState }; "routines.list": { params: RoutinesListParams; result: Array }; "routines.create": { params: RoutinesCreateParams; result: Routine }; "routines.update": { params: RoutinesUpdateParams; result: Routine }; @@ -1600,6 +1657,8 @@ export interface RpcNotifications { "reaction.changed": ReactionChanged; "bot.rules": BotRules; "bot.desktop": BotDesktop; + "desktop.changed": DesktopState; + "desktop.frame": DesktopFrame; "browser.changed": BrowserState; "browser.action": BrowserAction; "browser.frame": BrowserFrame; diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 8be5f5a9..29cc7b97 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -107,3 +107,63 @@ pub struct BotDesktop { /// Oldest first. pub grants: Vec, } + +/// The window a bot is using on the owner's desktop. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopWindow { + pub id: u64, + pub title: String, + /// The app's name for people. + pub app: String, +} + +/// What a bot does on the owner's desktop, for its panel (spec 24.9): the +/// params of `desktop.changed`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopState { + pub bot_id: BotId, + /// The window it read or acted in last. + pub window: Option, + /// What it did last, in English, as its tool said ("Clicked button + /// \"Save\"."); `null` after only reading. + pub action: Option, + /// When it last read or acted, Unix ms. + pub at: Option, + /// The owner stopped it (spec 24.9): its desktop tools refuse until + /// they let it go on. + pub stopped: bool, +} + +/// A picture of the window a bot is using, for whoever watches its panel. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopFrame { + pub bot_id: BotId, + /// JPEG, base64. + pub data: String, + pub width: u32, + pub height: u32, +} + +/// The result of `desktop.watch`: the bot's state and, when it uses a +/// window, its picture now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopView { + pub state: DesktopState, + pub frame: Option, +} + +/// `desktop.watch`, `desktop.stop` and `desktop.resume`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopBotParams { + pub bot_id: BotId, +} diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index 79cd1ef1..131f090b 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -12,6 +12,11 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); } pub(super) fn methods(out: &mut Out) { @@ -25,9 +30,23 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), &out.name::(), ); + let bot = out.name::(); + out.method(method::DESKTOP_WATCH, &bot, &out.name::()); + out.method(method::DESKTOP_UNWATCH, "undefined", "null"); + let state = out.name::(); + out.method(method::DESKTOP_STOP, &bot, &state); + out.method(method::DESKTOP_RESUME, &bot, &state); } pub(super) fn notifications(out: &mut Out) { let desktop = out.name::(); let _ = writeln!(out.text, " \"{}\": {desktop};", notification::BOT_DESKTOP); + let state = out.name::(); + let _ = writeln!( + out.text, + " \"{}\": {state};", + notification::DESKTOP_CHANGED + ); + let frame = out.name::(); + let _ = writeln!(out.text, " \"{}\": {frame};", notification::DESKTOP_FRAME); } diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index f1a7ecb9..cac4565e 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -125,6 +125,10 @@ pub mod method { pub const RULES_DELETE: &str = "rules.delete"; pub const DESKTOP_GRANTS: &str = "desktop.grants"; pub const DESKTOP_REVOKE: &str = "desktop.revoke"; + pub const DESKTOP_WATCH: &str = "desktop.watch"; + pub const DESKTOP_UNWATCH: &str = "desktop.unwatch"; + pub const DESKTOP_STOP: &str = "desktop.stop"; + pub const DESKTOP_RESUME: &str = "desktop.resume"; pub const MESSAGES_SEND: &str = "messages.send"; pub const MESSAGES_LIST: &str = "messages.list"; pub const ATTACHMENTS_READ: &str = "attachments.read"; @@ -175,6 +179,9 @@ pub mod notification { pub const BOT_CONTEXT: &str = "bot.context"; pub const BOT_RULES: &str = "bot.rules"; pub const BOT_DESKTOP: &str = "bot.desktop"; + pub const DESKTOP_CHANGED: &str = "desktop.changed"; + /// Only to the connection watching that bot's desktop panel (spec 24.9). + pub const DESKTOP_FRAME: &str = "desktop.frame"; pub const CHAT_ITEM: &str = "chat.item"; pub const CHAT_DELTA: &str = "chat.delta"; pub const MESSAGE_CREATED: &str = "message.created"; diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs new file mode 100644 index 00000000..67bad06c --- /dev/null +++ b/crates/botloftd/src/desktop/activity.rs @@ -0,0 +1,128 @@ +//! What each bot does on the owner's desktop, for its panel (spec 24.9): +//! the window it is using, what it did last, and whether the owner stopped +//! it. A stopped bot's desktop tools refuse until the owner lets it go on. + +use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{DesktopState, DesktopWindow}; + +use crate::platform::desktop::Window; + +/// What the bot reads when the owner stopped it. +pub const STOPPED: &str = "The owner stopped you from using their desktop. Do not try again: \ + tell them in the chat what you were doing, and they let you go on when they want."; + +#[derive(Default)] +pub struct Activity { + by_bot: Mutex>, +} + +fn blank(bot: &BotId) -> DesktopState { + DesktopState { + bot_id: bot.clone(), + window: None, + action: None, + at: None, + stopped: false, + } +} + +impl Activity { + fn change(&self, bot: &BotId, how: impl FnOnce(&mut DesktopState)) -> DesktopState { + let mut all = self.by_bot.lock().unwrap_or_else(PoisonError::into_inner); + let state = all.entry(bot.clone()).or_insert_with(|| blank(bot)); + how(state); + state.clone() + } + + pub fn state(&self, bot: &BotId) -> DesktopState { + self.by_bot + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .cloned() + .unwrap_or_else(|| blank(bot)) + } + + /// The window the bot is using now, by its number. + pub fn window(&self, bot: &BotId) -> Option { + self.state(bot).window.map(|window| window.id) + } + + /// The bot read or acted in `window` at `now`; `action` says what it + /// did, when it did something. + pub fn used( + &self, + bot: &BotId, + window: &Window, + action: Option, + now: i64, + ) -> DesktopState { + self.change(bot, |state| { + state.window = Some(DesktopWindow { + id: window.id, + title: window.title.clone(), + app: window.app.name.clone(), + }); + state.action = action; + state.at = Some(now); + }) + } + + pub fn stopped(&self, bot: &BotId) -> bool { + self.state(bot).stopped + } + + /// Stops the bot, or lets it go on. + pub fn set_stopped(&self, bot: &BotId, stopped: bool) -> DesktopState { + self.change(bot, |state| state.stopped = stopped) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::platform::desktop::App; + + #[test] + fn a_bot_shows_its_window_and_last_action_and_stays_stopped_until_it_may_go_on() { + let activity = Activity::default(); + let bot = BotId::generate(); + assert_eq!(activity.state(&bot), blank(&bot)); + let window = Window { + id: 7, + title: "Visits".to_owned(), + class: "Any".to_owned(), + app: App { + path: PathBuf::from(r"C:\Apps\clinic.exe"), + name: "Clinic".to_owned(), + }, + minimized: false, + elevated: false, + }; + let used = activity.used( + &bot, + &window, + Some("Clicked button \"Save\".".to_owned()), + 9, + ); + assert_eq!(activity.window(&bot), Some(7)); + assert_eq!( + used.window.map(|window| window.app), + Some("Clinic".to_owned()) + ); + assert_eq!(used.at, Some(9)); + + assert!(!activity.stopped(&bot)); + assert!(activity.set_stopped(&bot, true).stopped); + assert!(activity.stopped(&bot)); + // Reading again does not let it go on. + activity.used(&bot, &window, None, 10); + assert!(activity.stopped(&bot)); + assert!(!activity.set_stopped(&bot, false).stopped); + } +} diff --git a/crates/botloftd/src/desktop.rs b/crates/botloftd/src/desktop/mod.rs similarity index 83% rename from crates/botloftd/src/desktop.rs rename to crates/botloftd/src/desktop/mod.rs index cd8f1b85..f1031c55 100644 --- a/crates/botloftd/src/desktop.rs +++ b/crates/botloftd/src/desktop/mod.rs @@ -1,6 +1,13 @@ //! The owner's desktop as the bots use it (spec 24): one action at a time -//! across every bot, whether the owner is there to see it, and each bot's -//! last reading, whose refs its actions name. +//! across every bot, whether the owner is there to see it, each bot's last +//! reading, whose refs its actions name, and what each bot does there, for +//! its panel, live. + +mod activity; +mod screen; + +pub use activity::{Activity, STOPPED}; +pub use screen::DesktopWatching; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -8,8 +15,10 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopView; use tokio::sync::MutexGuard; +use self::screen::Screens; use crate::platform::desktop::Control; /// How long the owner may go without touching the computer and still count @@ -34,6 +43,8 @@ pub struct Desktop { /// Held for each desktop action: the cursor is one. turn: tokio::sync::Mutex<()>, idle: Mutex, + pub activity: Arc, + screens: Arc, } /// Why the bot may not use the desktop now. @@ -68,6 +79,8 @@ impl Desktop { readings: Mutex::new(HashMap::new()), turn: tokio::sync::Mutex::new(()), idle: Mutex::new(idle), + activity: Arc::default(), + screens: Arc::default(), } } @@ -119,6 +132,19 @@ impl Desktop { .cloned() } + /// Starts watching the window the bot is using (spec 24.9). + pub fn watch(&self, bot: &BotId) -> DesktopWatching { + self.screens.watch(bot, &self.activity) + } + + /// The bot's state and the newest picture, for a panel that opens. + pub fn view(&self, bot: &BotId) -> DesktopView { + DesktopView { + state: self.activity.state(bot), + frame: self.screens.now(bot), + } + } + /// Waits for the desktop to be free; it is the caller's until dropped. pub async fn turn(&self) -> MutexGuard<'_, ()> { self.turn.lock().await diff --git a/crates/botloftd/src/desktop/screen.rs b/crates/botloftd/src/desktop/screen.rs new file mode 100644 index 00000000..5f99e161 --- /dev/null +++ b/crates/botloftd/src/desktop/screen.rs @@ -0,0 +1,100 @@ +//! The live picture of the window a bot is using, for whoever watches its +//! desktop panel (spec 24.9). While someone watches, the window is +//! pictured four times a second and a picture goes out only when it +//! changed; when the last one stops, so does the picturing. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopFrame; +use tokio::sync::watch; + +use super::activity::Activity; +use crate::platform::desktop; + +/// How often a watched window is pictured. +const EVERY: Duration = Duration::from_millis(250); + +type Frames = watch::Sender>>; + +/// Held by the connection; dropping it stops watching. +pub struct DesktopWatching { + pub bot: BotId, + /// The newest picture; `changed()` wakes on each new one. + pub frames: watch::Receiver>>, +} + +#[derive(Default)] +pub struct Screens { + /// Each watched bot's pictures, while its picturing runs. + running: Mutex>, +} + +impl Screens { + /// Starts watching the bot's window: its picturing runs while anyone + /// watches. + pub fn watch(self: &Arc, bot: &BotId, activity: &Arc) -> DesktopWatching { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(frames) = running.get(bot) { + return DesktopWatching { + bot: bot.clone(), + frames: frames.subscribe(), + }; + } + let (frames, watching) = watch::channel(None); + running.insert(bot.clone(), frames.clone()); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let (screens, activity, bot) = (Arc::clone(self), Arc::clone(activity), bot.clone()); + runtime.spawn(async move { screens.picture(&bot, &activity, &frames).await }); + } + DesktopWatching { + bot: bot.clone(), + frames: watching, + } + } + + /// The newest picture of the bot's window, while it is watched. + pub fn now(&self, bot: &BotId) -> Option { + self.running + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .and_then(|frames| frames.borrow().as_deref().cloned()) + } + + /// Pictures the bot's window until nobody watches. + async fn picture(&self, bot: &BotId, activity: &Activity, frames: &Frames) { + let mut last: Option = None; + loop { + { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if frames.receiver_count() == 0 { + running.remove(bot); + return; + } + } + if let Some(id) = activity.window(bot) { + let picture = tokio::task::spawn_blocking(move || desktop::picture(id)).await; + if let Ok(Ok(picture)) = picture + && !picture.jpeg.is_empty() + { + let data = BASE64.encode(&picture.jpeg); + if last.as_deref() != Some(data.as_str()) { + last = Some(data.clone()); + frames.send_replace(Some(Arc::new(DesktopFrame { + bot_id: bot.clone(), + data, + width: picture.width, + height: picture.height, + }))); + } + } + } + tokio::time::sleep(EVERY).await; + } + } +} diff --git a/crates/botloftd/src/main.rs b/crates/botloftd/src/main.rs index e60871c2..c14c0660 100644 --- a/crates/botloftd/src/main.rs +++ b/crates/botloftd/src/main.rs @@ -22,7 +22,7 @@ use botloftd::trash::RecycleBin; use botloftd::{approvals, autostart, keep_awake, logging, routines, secrets, server}; use clap::{Parser, Subcommand}; use tokio::net::TcpListener; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[derive(Parser)] #[command(name = "botloftd", version, about = "Botloft daemon")] @@ -137,6 +137,12 @@ fn run(paths: Paths, config: Config, config_path: PathBuf, scheduled: bool) -> a }); // No bot process survived the last run, so nobody waits for these. approvals::expire_all(&daemon); + // Ctrl+Alt+End stops every bot on the desktop, app open or not (spec 24.9). + let stopping = Arc::clone(&daemon); + match platform::desktop::on_stop_key(move || botloftd::service::desktop::stop_all(&stopping)) { + Ok(()) | Err(platform::desktop::DesktopError::Unavailable) => {} + Err(err) => warn!("the shortcut that stops bots on the desktop is not available: {err}"), + } let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 3de60059..73096570 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -126,6 +126,20 @@ pub fn windows() -> Result, DesktopError> { } } +/// Calls `stop` each time the owner presses Ctrl+Alt+End, from now on +/// (spec 24.9). Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + #[cfg(windows)] + { + windows::on_stop_key(stop) + } + #[cfg(not(windows))] + { + let _ = stop; + Err(DesktopError::Unavailable) + } +} + /// How long ago the owner last used the mouse or keyboard (spec 24.8); /// `None` where that cannot be known. pub fn owner_idle() -> Option { diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 64207d3c..c7c82c76 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -8,6 +8,7 @@ mod act_tests; mod capture; mod input; mod process; +mod stop_key; #[cfg(test)] mod test_window; #[cfg(test)] @@ -31,6 +32,7 @@ use windows::core::{BOOL, w}; pub use act::act; pub use capture::capture; pub use input::owner_idle; +pub use stop_key::on_stop_key; pub use uia::read; use super::{App, DesktopError, Window}; diff --git a/crates/botloftd/src/platform/desktop/windows/stop_key.rs b/crates/botloftd/src/platform/desktop/windows/stop_key.rs new file mode 100644 index 00000000..b70cde46 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/stop_key.rs @@ -0,0 +1,52 @@ +//! The shortcut that stops every bot on the desktop, Ctrl+Alt+End (spec +//! 24.9): registered for the whole session on a thread of its own, so it +//! works with the app closed. + +use std::sync::mpsc; + +use windows::Win32::UI::Input::KeyboardAndMouse::{ + MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, RegisterHotKey, VK_END, +}; +use windows::Win32::UI::WindowsAndMessaging::{GetMessageW, MSG, WM_HOTKEY}; + +use super::super::DesktopError; + +/// The shortcut's id for this thread. +const STOP: i32 = 1; + +/// Calls `stop` each time the owner presses the shortcut, from now on. +/// Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + let (told, registered) = mpsc::channel(); + std::thread::Builder::new() + .name("desktop-stop-key".to_owned()) + .spawn(move || { + // SAFETY: a shortcut for this thread's own message queue. + let held = unsafe { + RegisterHotKey( + None, + STOP, + MOD_CONTROL | MOD_ALT | MOD_NOREPEAT, + u32::from(VK_END.0), + ) + }; + let ok = held.is_ok(); + let _ = told.send(held.map_err(|err| DesktopError::System(err.message()))); + if !ok { + return; + } + let mut message = MSG::default(); + // SAFETY: this thread's own queue, read until the process ends. + while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() { + if message.message == WM_HOTKEY && message.wParam.0 == STOP as usize { + stop(); + } + } + }) + .map_err(|err| DesktopError::System(err.to_string()))?; + registered.recv().unwrap_or_else(|_| { + Err(DesktopError::System( + "the shortcut's thread ended".to_owned(), + )) + }) +} diff --git a/crates/botloftd/src/rpc/desktop_watch.rs b/crates/botloftd/src/rpc/desktop_watch.rs new file mode 100644 index 00000000..9390c5f6 --- /dev/null +++ b/crates/botloftd/src/rpc/desktop_watch.rs @@ -0,0 +1,50 @@ +//! A connection watching one bot's desktop panel (spec 24.9): +//! `desktop.watch` and `desktop.unwatch`, and the pictures that follow. + +use botloft_core::protocol::{DesktopBotParams, error_code, method, notification}; +use serde_json::Value; + +use super::dispatch::parse; +use super::jsonrpc::{self, RpcError}; +use crate::desktop::DesktopWatching; +use crate::service::bots; +use crate::state::Daemon; + +#[derive(Default)] +pub(super) struct DesktopWatch { + current: Option, +} + +impl DesktopWatch { + pub(super) fn request( + &mut self, + daemon: &Daemon, + name: &str, + params: Option, + ) -> Result { + // Stop the one before, even if the new one is refused. + self.current = None; + if name == method::DESKTOP_UNWATCH { + return Ok(Value::Null); + } + let params: DesktopBotParams = parse(params)?; + bots::find(&daemon.store(), ¶ms.bot_id).map_err(RpcError::from)?; + self.current = Some(daemon.desktop.watch(¶ms.bot_id)); + serde_json::to_value(daemon.desktop.view(¶ms.bot_id)) + .map_err(|err| RpcError::new(error_code::INTERNAL_ERROR, err.to_string())) + } + + /// The watched window's next picture, as a notification. Never resolves + /// while nothing is watched. + pub(super) async fn next_frame(&mut self) -> Option { + let Some(watching) = self.current.as_mut() else { + return std::future::pending().await; + }; + if watching.frames.changed().await.is_err() { + self.current = None; + return None; + } + let frame = watching.frames.borrow_and_update().clone()?; + Some(jsonrpc::notification(notification::DESKTOP_FRAME, &*frame)) + } +} diff --git a/crates/botloftd/src/rpc/dispatch.rs b/crates/botloftd/src/rpc/dispatch.rs index f1c64d06..035111b5 100644 --- a/crates/botloftd/src/rpc/dispatch.rs +++ b/crates/botloftd/src/rpc/dispatch.rs @@ -51,6 +51,8 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option) -> Result reply(rules::delete(daemon, parse(params)?)), method::DESKTOP_GRANTS => reply(desktop::grants(daemon, parse(params)?)), method::DESKTOP_REVOKE => reply(desktop::revoke(daemon, parse(params)?)), + method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)), + method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)), method::MESSAGES_SEND => reply(messages::send(daemon, parse(params)?)), method::MESSAGES_LIST => reply(messages::list(daemon, parse(params)?)), method::ATTACHMENTS_READ => reply(attachments::read(daemon, parse(params)?)), diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index d92ba5cb..3085aa46 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -1,24 +1,26 @@ //! The app's connection: one WebSocket per client, JSON-RPC 2.0 over text //! frames (spec 11). The first request must be `session.hello`. +mod desktop_watch; mod dispatch; pub mod jsonrpc; +mod notify; mod watching; use std::sync::Arc; use std::time::Duration; use axum::extract::ws::{Message, WebSocket}; -use botloft_core::protocol::{ - HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method, notification, -}; +use botloft_core::protocol::{HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method}; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use serde_json::Value; use tokio::sync::{broadcast, mpsc, watch}; use tracing::{debug, warn}; +use self::desktop_watch::DesktopWatch; use self::jsonrpc::RpcError; +use self::notify::to_notification; use self::watching::Watch; use crate::state::{Daemon, Event}; @@ -140,10 +142,11 @@ async fn run( daemon: &Arc, ) { let mut watch = Watch::default(); + let mut desktop = DesktopWatch::default(); loop { let frame = tokio::select! { message = stream.next() => match message { - Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, frames) { + Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, &mut desktop, frames) { Routed::Reply(frame) => frame, Routed::InOrder(request) => answer(daemon, request).await, Routed::Aside(request) => { @@ -167,6 +170,10 @@ async fn run( frames.send_replace(frame); None } + frame = desktop.next_frame() => { + frames.send_replace(frame); + None + } }; if let Some(frame) = frame && outbox.send(frame).await.is_err() @@ -201,6 +208,7 @@ fn route( daemon: &Arc, text: &str, watch: &mut Watch, + desktop: &mut DesktopWatch, frames: &watch::Sender>, ) -> Routed { let request = match jsonrpc::parse(text) { @@ -228,6 +236,11 @@ fn route( | method::BROWSER_SWITCH_TAB | method::BROWSER_OPEN | method::BROWSER_TEACH => watch.request(daemon, &request.method, request.params), + // So is watching a bot's desktop panel (spec 24.9). + method::DESKTOP_WATCH | method::DESKTOP_UNWATCH => { + frames.send_replace(None); + desktop.request(daemon, &request.method, request.params) + } // A window of its own outlives the request (spec 21.11). method::BROWSER_WINDOW => watch.window(daemon, request.params), name if ASIDE.contains(&name) => return Routed::Aside(request), @@ -271,30 +284,3 @@ fn reply(id: &Value, result: Result) -> String { Err(err) => jsonrpc::failure(id, &err), } } - -fn to_notification(event: &Event) -> String { - use jsonrpc::notification as note; - match event { - Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), - Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), - Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), - Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), - Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), - Event::BotState(state) => note(notification::BOT_STATE, state), - Event::BotContext(context) => note(notification::BOT_CONTEXT, context), - Event::BotRules(rules) => note(notification::BOT_RULES, rules), - Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), - Event::ChatItem(item) => note(notification::CHAT_ITEM, item), - Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), - Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), - Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), - Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), - Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), - Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), - Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), - Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), - Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), - Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), - Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), - } -} diff --git a/crates/botloftd/src/rpc/notify.rs b/crates/botloftd/src/rpc/notify.rs new file mode 100644 index 00000000..1c2d65f5 --- /dev/null +++ b/crates/botloftd/src/rpc/notify.rs @@ -0,0 +1,34 @@ +//! What changed, as the notification the apps get (spec 11). + +use botloft_core::protocol::notification; + +use super::jsonrpc; +use crate::state::Event; + +pub(super) fn to_notification(event: &Event) -> String { + use jsonrpc::notification as note; + match event { + Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), + Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), + Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), + Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), + Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), + Event::BotState(state) => note(notification::BOT_STATE, state), + Event::BotContext(context) => note(notification::BOT_CONTEXT, context), + Event::BotRules(rules) => note(notification::BOT_RULES, rules), + Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), + Event::DesktopChanged(state) => note(notification::DESKTOP_CHANGED, state), + Event::ChatItem(item) => note(notification::CHAT_ITEM, item), + Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), + Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), + Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), + Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), + Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), + Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), + Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), + Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), + Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), + Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), + Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), + } +} diff --git a/crates/botloftd/src/service/desktop.rs b/crates/botloftd/src/service/desktop.rs index 2bdc2b7c..4d44e656 100644 --- a/crates/botloftd/src/service/desktop.rs +++ b/crates/botloftd/src/service/desktop.rs @@ -1,9 +1,15 @@ //! What each bot may see and do on the owner's desktop (spec 24.2, //! 24.10): `desktop.grants` and `desktop.revoke`. Grants come from the -//! owner answering a bot's request (`tools/desktop.rs`). +//! owner answering a bot's request (`tools/desktop.rs`). And stopping a +//! bot there, or every bot at once (spec 24.9): `desktop.stop` and +//! `desktop.resume`. use botloft_core::ids::BotId; -use botloft_core::protocol::{BotDesktop, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams}; +use botloft_core::protocol::{ + BotDesktop, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams, + DesktopState, +}; +use tracing::{info, warn}; use super::{ApiError, ApiResult}; use crate::state::{Daemon, Event}; @@ -34,3 +40,39 @@ pub(crate) fn changed(daemon: &Daemon, bot_id: &BotId) -> ApiResult daemon.emit(Event::BotDesktop(desktop.clone())); Ok(desktop) } + +/// Stops the bot on the desktop, or lets it go on. +fn set_stopped(daemon: &Daemon, bot: &BotId, stopped: bool) -> DesktopState { + let state = daemon.desktop.activity.set_stopped(bot, stopped); + daemon.emit(Event::DesktopChanged(state.clone())); + state +} + +pub fn stop(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, true)) +} + +pub fn resume(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, false)) +} + +/// Stops every bot on the desktop: the owner pressed the stop shortcut +/// (spec 24.9). +pub fn stop_all(daemon: &Daemon) { + let bots = match daemon.store().bots(None, false) { + Ok(bots) => bots, + Err(err) => { + warn!("could not list the bots to stop them on the desktop: {err}"); + return; + } + }; + for bot in &bots { + set_stopped(daemon, &bot.id, true); + } + info!( + bots = bots.len(), + "the owner stopped every bot on the desktop" + ); +} diff --git a/crates/botloftd/src/state.rs b/crates/botloftd/src/state.rs index b9768d08..f4578162 100644 --- a/crates/botloftd/src/state.rs +++ b/crates/botloftd/src/state.rs @@ -8,7 +8,8 @@ use std::time::Instant; use botloft_core::protocol::{ AccountUsage, Bot, BotContextChanged, BotDeleted, BotDesktop, BotRules, BotStateChanged, BrowserAction, BrowserState, ChatDelta, ChatItemChanged, Crew, CrewDeleted, Delivery, - FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, ScreenDraft, Task, + DesktopState, FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, + ScreenDraft, Task, }; use botloft_store::Store; use tokio::runtime::{Handle, RuntimeFlavor}; @@ -43,6 +44,7 @@ pub enum Event { BotContext(BotContextChanged), BotRules(BotRules), BotDesktop(BotDesktop), + DesktopChanged(DesktopState), ChatItem(ChatItemChanged), ChatDelta(ChatDelta), MessageCreated(Message), diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index 3867e14e..08464846 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -15,8 +15,10 @@ use super::calls::explain; use super::desktop_act::{Ask, act}; use super::desktop_grant::{blocking, granted, heading}; use super::desktop_list::list; +use crate::desktop::STOPPED; +use crate::platform::desktop::Window; use crate::platform::desktop::{self as platform, Action, Scroll, render}; -use crate::state::Daemon; +use crate::state::{Daemon, Event}; pub(super) const PREFIX: &str = "desktop_"; @@ -136,6 +138,9 @@ pub(super) async fn call( } async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Result { + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } daemon .desktop .owner_here() @@ -161,10 +166,14 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let controls = blocking(move || platform::read(id)).await?; let (text, next) = render(&controls, from); daemon.desktop.keep(bot, id, controls); + used(daemon, bot, &window, None); let mut reading = format!("{}\n{text}", heading(&window)); if let Some(next) = next { reading.push_str(&format!( @@ -186,8 +195,12 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let picture = blocking(move || platform::picture(id)).await?; + used(daemon, bot, &window, None); if picture.jpeg.is_empty() { return Err("The picture of that window came out empty.".to_owned()); } @@ -201,3 +214,10 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul .map(Reply::Text), } } + +/// The bot read or acted in `window`: its panel shows it (spec 24.9). +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { + let now = daemon.clock.now_ms(); + let state = daemon.desktop.activity.used(bot, window, action, now); + daemon.emit(Event::DesktopChanged(state)); +} diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index 29db9662..cf1cec8c 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -8,7 +8,9 @@ use std::time::Duration; use botloft_core::ids::BotId; use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use super::desktop::used; use super::desktop_grant::{blocking, describe, granted, heading}; +use crate::desktop::STOPPED; use crate::platform::desktop::{ self as platform, Acted, Action, Control, DesktopError, Window, by_reference, render, }; @@ -73,6 +75,10 @@ pub(super) async fn act( ) .await?; let _turn = daemon.desktop.turn().await; + // The owner may have stopped it while it waited. + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let (id, target, action) = (window.id, control.runtime_id.clone(), ask.action.clone()); let acted = blocking(move || platform::act(id, &target, &action)).await?; tokio::time::sleep(SETTLE).await; @@ -86,6 +92,7 @@ pub(super) async fn act( let (text, next) = render(&controls, 0); daemon.desktop.keep(bot, id, controls); let mut answer = done(&ask.action, &control); + used(daemon, bot, &window, Some(answer.clone())); if acted == Acted::Waiting { answer.push_str( " The app did not answer in time: it may be showing a dialog. Call desktop_windows \ diff --git a/crates/botloftd/tests/common/desktop.rs b/crates/botloftd/tests/common/desktop.rs new file mode 100644 index 00000000..ca9415ec --- /dev/null +++ b/crates/botloftd/tests/common/desktop.rs @@ -0,0 +1,169 @@ +//! The desktop tests' own window (spec 24): a Save button that renames it, +//! a field and a password field, on a thread with its message loop; and a +//! daemon with a crew and a bot ready to use it. Windows only. + +use std::sync::mpsc; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::{Value, json}; +use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::WindowsAndMessaging::{ + BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, + GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, + TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, + WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, +}; +use windows::core::{PCWSTR, w}; + +use super::bots::ready_bot; +use super::mcp::Mcp; +use super::{Client, TestDaemon}; + +pub const TITLE: &str = "Botloft desktop: the clinic"; + +unsafe extern "system" fn procedure( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if message == WM_DESTROY { + // SAFETY: ends this thread's message loop. + unsafe { PostQuitMessage(0) }; + return LRESULT(0); + } + // The Save button (id 1) clicked: the window says so in its title. + if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { + // SAFETY: renames our own window, on its thread. + let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; + return LRESULT(0); + } + // SAFETY: everything else as Windows does by default. + unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } +} + +/// A window of the test's own, on a thread with its message loop. +pub struct TestWindow { + pub id: u64, + thread: Option>, +} + +impl TestWindow { + pub fn open() -> Self { + let (sent, opened) = mpsc::channel(); + let thread = std::thread::spawn(move || { + // SAFETY: a window and its controls on this thread, with its loop. + unsafe { + let class = WNDCLASSW { + lpfnWndProc: Some(procedure), + hInstance: GetModuleHandleW(None).expect("module").into(), + lpszClassName: w!("BotloftDesktopToolsTest"), + ..Default::default() + }; + RegisterClassW(&class); + let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + w!("BotloftDesktopToolsTest"), + PCWSTR(title.as_ptr()), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + CW_USEDEFAULT, + CW_USEDEFAULT, + 420, + 320, + None, + None, + None, + None, + ) + .expect("window"); + let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { + CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + text, + WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), + 10, + top, + 160, + 24, + Some(hwnd), + Some(HMENU(id as *mut std::ffi::c_void)), + None, + None, + ) + .expect("control"); + }; + child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); + child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); + child( + w!("EDIT"), + w!("hunter2"), + WS_BORDER.0 | ES_PASSWORD as u32, + 70, + 3, + ); + sent.send(hwnd.0 as usize as u64).expect("send"); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + }); + let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); + std::thread::sleep(Duration::from_millis(300)); + Self { + id, + thread: Some(thread), + } + } +} + +impl Drop for TestWindow { + fn drop(&mut self) { + let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); + // SAFETY: asks the window's own thread to close it. + let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +pub struct Setup { + pub t: TestDaemon, + pub app: Client, + pub bot: Value, + pub mcp: Mcp, +} + +pub async fn setup() -> Setup { + let t = TestDaemon::start_supervised().await; + let mut app = t.session().await; + let crew = app + .call("crews.create", json!({ "name": "Clinic" })) + .await + .expect("crew"); + let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; + Setup { t, app, bot, mcp } +} + +pub fn reading(result: &Result) -> &str { + match result { + Ok(text) | Err(text) => text, + } +} + +/// The ref of the line that has `what` in a reading. +pub fn ref_in(reading: &str, what: &str) -> String { + let line = reading + .lines() + .find(|line| line.contains(what)) + .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); + let start = line.find('[').expect("a ref") + 1; + let end = line.find(']').expect("a ref"); + line[start..end].to_owned() +} diff --git a/crates/botloftd/tests/common/mod.rs b/crates/botloftd/tests/common/mod.rs index 7a9dfe84..bf81669f 100644 --- a/crates/botloftd/tests/common/mod.rs +++ b/crates/botloftd/tests/common/mod.rs @@ -6,6 +6,8 @@ pub mod bots; pub mod browsing; pub mod context; +#[cfg(windows)] +pub mod desktop; pub mod mcp; pub mod routines; pub mod site; diff --git a/crates/botloftd/tests/desktop.rs b/crates/botloftd/tests/desktop.rs index 466b3710..0e1a405f 100644 --- a/crates/botloftd/tests/desktop.rs +++ b/crates/botloftd/tests/desktop.rs @@ -6,161 +6,11 @@ mod common; -use std::sync::mpsc; -use std::thread::JoinHandle; use std::time::Duration; -use common::Client; -use common::TestDaemon; -use common::bots::ready_bot; use common::browsing::{call, pending_approval}; -use common::mcp::Mcp; -use serde_json::{Value, json}; -use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; -use windows::Win32::System::LibraryLoader::GetModuleHandleW; -use windows::Win32::UI::WindowsAndMessaging::{ - BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, - GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, - TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, - WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, -}; -use windows::core::{PCWSTR, w}; - -const TITLE: &str = "Botloft desktop: the clinic"; - -unsafe extern "system" fn procedure( - hwnd: HWND, - message: u32, - wparam: WPARAM, - lparam: LPARAM, -) -> LRESULT { - if message == WM_DESTROY { - // SAFETY: ends this thread's message loop. - unsafe { PostQuitMessage(0) }; - return LRESULT(0); - } - // The Save button (id 1) clicked: the window says so in its title. - if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { - // SAFETY: renames our own window, on its thread. - let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; - return LRESULT(0); - } - // SAFETY: everything else as Windows does by default. - unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } -} - -/// A window of the test's own, on a thread with its message loop. -struct TestWindow { - id: u64, - thread: Option>, -} - -impl TestWindow { - fn open() -> Self { - let (sent, opened) = mpsc::channel(); - let thread = std::thread::spawn(move || { - // SAFETY: a window and its controls on this thread, with its loop. - unsafe { - let class = WNDCLASSW { - lpfnWndProc: Some(procedure), - hInstance: GetModuleHandleW(None).expect("module").into(), - lpszClassName: w!("BotloftDesktopToolsTest"), - ..Default::default() - }; - RegisterClassW(&class); - let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); - let hwnd = CreateWindowExW( - WINDOW_EX_STYLE::default(), - w!("BotloftDesktopToolsTest"), - PCWSTR(title.as_ptr()), - WS_OVERLAPPEDWINDOW | WS_VISIBLE, - CW_USEDEFAULT, - CW_USEDEFAULT, - 420, - 320, - None, - None, - None, - None, - ) - .expect("window"); - let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { - CreateWindowExW( - WINDOW_EX_STYLE::default(), - class, - text, - WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), - 10, - top, - 160, - 24, - Some(hwnd), - Some(HMENU(id as *mut std::ffi::c_void)), - None, - None, - ) - .expect("control"); - }; - child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); - child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); - child( - w!("EDIT"), - w!("hunter2"), - WS_BORDER.0 | ES_PASSWORD as u32, - 70, - 3, - ); - sent.send(hwnd.0 as usize as u64).expect("send"); - let mut message = MSG::default(); - while GetMessageW(&mut message, None, 0, 0).as_bool() { - let _ = TranslateMessage(&message); - DispatchMessageW(&message); - } - } - }); - let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); - std::thread::sleep(Duration::from_millis(300)); - Self { - id, - thread: Some(thread), - } - } -} - -impl Drop for TestWindow { - fn drop(&mut self) { - let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); - // SAFETY: asks the window's own thread to close it. - let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } - } -} - -struct Setup { - t: TestDaemon, - app: Client, - bot: Value, - mcp: Mcp, -} - -async fn setup() -> Setup { - let t = TestDaemon::start_supervised().await; - let mut app = t.session().await; - let crew = app - .call("crews.create", json!({ "name": "Clinic" })) - .await - .expect("crew"); - let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; - Setup { t, app, bot, mcp } -} - -fn reading(result: &Result) -> &str { - match result { - Ok(text) | Err(text) => text, - } -} +use common::desktop::{TITLE, TestWindow, reading, ref_in, setup}; +use serde_json::json; #[tokio::test(flavor = "multi_thread")] async fn a_bot_sees_an_app_once_the_owner_lets_it_and_never_a_password() { @@ -251,17 +101,6 @@ async fn nothing_is_seen_while_the_owner_is_away() { ); } -/// The ref of the line that has `what` in a reading. -fn ref_in(reading: &str, what: &str) -> String { - let line = reading - .lines() - .find(|line| line.contains(what)) - .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); - let start = line.find('[').expect("a ref") + 1; - let end = line.find(']').expect("a ref"); - line[start..end].to_owned() -} - #[tokio::test(flavor = "multi_thread")] async fn a_bot_uses_an_app_once_the_owner_lets_it_and_never_types_a_password() { let window = TestWindow::open(); diff --git a/crates/botloftd/tests/desktop_panel.rs b/crates/botloftd/tests/desktop_panel.rs new file mode 100644 index 00000000..b4f2cc7b --- /dev/null +++ b/crates/botloftd/tests/desktop_panel.rs @@ -0,0 +1,83 @@ +//! A bot's desktop panel with real windows (spec 24.9): the window it is +//! using and what it did, its pictures while the panel is open, and the +//! owner stopping it, one bot or all of them. Windows only. +#![cfg(windows)] + +mod common; + +use common::browsing::{call, pending_approval}; +use common::desktop::{TITLE, TestWindow, reading, setup}; +use serde_json::json; + +#[tokio::test(flavor = "multi_thread")] +async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { + let window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + // Nothing used yet: no window, no picture. + let view = s + .app + .call("desktop.watch", id.clone()) + .await + .expect("watch"); + assert_eq!(view["state"]["window"], serde_json::Value::Null); + assert_eq!(view["state"]["stopped"], false); + assert_eq!(view["frame"], serde_json::Value::Null); + + let look = json!({ "window": window.id, "why": "To read the visit" }); + let looking = call(&s.mcp, "desktop_look", look); + let asked = pending_approval(&mut s.app).await; + let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); + s.app.call("approvals.answer", allow).await.expect("answer"); + looking.await.expect("task").expect("reading"); + + // The panel hears which window, and its picture follows. + let changed = s.app.notification("desktop.changed").await; + assert_eq!(changed["window"]["title"], TITLE); + assert_eq!(changed["action"], serde_json::Value::Null); + let frame = s.app.notification("desktop.frame").await; + assert_eq!(frame["botId"], s.bot["id"]); + assert!(frame["data"].as_str().is_some_and(|data| data.len() > 100)); + + s.app + .call("desktop.unwatch", serde_json::Value::Null) + .await + .expect("unwatch"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_stopped_bot_cannot_use_the_desktop_until_the_owner_lets_it_go_on() { + let _window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + let stopped = s.app.call("desktop.stop", id.clone()).await.expect("stop"); + assert_eq!(stopped["stopped"], true); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); + + let going = s + .app + .call("desktop.resume", id.clone()) + .await + .expect("resume"); + assert_eq!(going["stopped"], false); + s.mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows again"); + + // The shortcut stops every bot at once. + botloftd::service::desktop::stop_all(&s.t.daemon); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); +} diff --git a/docs/spec.md b/docs/spec.md index 7dcb5f18..2c64c4a2 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1653,7 +1653,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 e D2 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1 e D2 implementados; D3 em andamento** (24.12): o daemon já guarda o que cada bot faz no desktop, manda a foto ao vivo, para e deixa continuar, e tem o atalho; falta o painel no app. É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1755,8 +1755,11 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela em que o bot está agindo (`PrintWindow` a cada mudança, no máximo 5 por segundo, só enquanto o painel está aberto), o nome do app, e a linha do que ele acabou de fazer. Um botão **Parar** encerra o que o bot faz no desktop na hora. -- **Parar tudo**: um atalho global, `Ctrl+Alt+End`, que o daemon registra (`RegisterHotKey`), para qualquer ação de desktop de qualquer bot, mesmo com o app fechado. +- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação como a tool disse (`Clicked button "Save".`; nada depois de só ler), quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. +- **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. +- **Parar tudo:** o atalho global `Ctrl+Alt+End`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele. - O cartão de permissão no chat (24.2) diz " quer ver " com o ícone de tela, o porquê do bot (" diz: ..."), o caminho do executável e o que ver deixa fazer (ler as janelas do app e fotografá-las com o dono no computador, nunca senhas, e onde tirar). Respondido, vira uma linha: " pode ver ", " não pode ver " com a nota, ou sem resposta. As tools `desktop_*` aparecem no chat com o ícone de tela e o porquê como resumo. ### 24.10 Permissões no app @@ -1784,7 +1787,7 @@ No servidor `botloft` (11), como as do navegador. |---|---|---| | **D1** Ver | `desktop_grants`, permissões pelo chat (app, Ver), o que nunca é liberado, `desktop_windows`, `desktop_look`, `desktop_screenshot` de uma janela, a seção Desktop nas configurações do bot | unidade; Windows de verdade com uma janela de teste própria (Win32 com botões e campos, inclusive de senha) | | **D2** Mexer | nível Mexer, `desktop_click`, `desktop_type`, `desktop_select`, `desktop_scroll` por acessibilidade, uma ação por vez | a janela de teste e o Bloco de Notas | -| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | app com FakeBotloft; o atalho no Windows de verdade | +| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | o daemon com uma janela de verdade (foto ao vivo, parar, deixar continuar, parar todos); app com FakeBotloft; o atalho apertado de verdade, à mão | | **D4** Mouse e teclado | a opção, `SendInput`, `desktop_press`, `desktop_click_at`, o dono assumindo pelo gancho, o aviso na tela | a janela de teste; o dono simulado por entrada não injetada | | **D5** Sem você | a opção, a tela de riscos, o aviso na volta, a regra dos 5 minutos, tela bloqueada | unidade; teste manual com a tela bloqueada | | **D6** Desktop inteiro | o alcance, a foto da tela com as janelas bloqueadas cobertas | a janela de teste e uma bloqueada | From 085d7ac790dc72082517cd03c36b5d740ebc2442 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:45:23 -0300 Subject: [PATCH 2/3] fix(desktop): the panel words the bot's action itself, and no app holds a reading for minutes DesktopState.action is now structured (DesktopAction: click, type, select or scroll, the control's name and the option), so the app words it in the owner's language instead of showing the tool's English. Every UI Automation session, reading too, gets 5 seconds per call and to connect, and a walk stops at 15 seconds with what it read: right after a classic combo box took a choice through its default action, the window could answer slowly, and the next reading, without a limit, hung for minutes. The choice and scroll test waits up to 2 seconds for the window to show them, as the tool does; 12 runs in a row pass. --- app/src/lib/fakeDesktop.ts | 3 +- app/src/lib/protocol.gen.ts | 21 ++++++++++-- crates/botloft-core/src/protocol/desktop.rs | 28 ++++++++++++++-- .../src/protocol/export_desktop.rs | 2 ++ crates/botloftd/src/desktop/activity.rs | 16 +++++----- .../src/platform/desktop/windows/act.rs | 23 +++++-------- .../src/platform/desktop/windows/act_tests.rs | 32 +++++++++++++++---- .../src/platform/desktop/windows/uia.rs | 27 +++++++++++++--- crates/botloftd/src/tools/desktop.rs | 4 +-- crates/botloftd/src/tools/desktop_act.rs | 19 +++++++++-- docs/spec.md | 4 +-- 11 files changed, 131 insertions(+), 48 deletions(-) diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index fabd6cbc..6f0e1de1 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -8,6 +8,7 @@ import type { FakeBotloft, Handlers } from "./fake"; import { notFound } from "./fakeRules"; import type { BotId, + DesktopAction, DesktopFrame, DesktopGrant, DesktopLevel, @@ -74,7 +75,7 @@ export class FakeDesktop { } /** The bot read or acted in `window`; `action` says what it did. */ - use(botId: BotId, window: DesktopWindow, action: string | null = null): DesktopState { + use(botId: BotId, window: DesktopWindow, action: DesktopAction | null = null): DesktopState { return this.set(botId, { window, action, at: this.fake.now }); } diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index a5c29847..a54e9090 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -878,6 +878,22 @@ export type DesktopWindow = { id: number, title: string, */ app: string, }; +/** + * What a bot did in a window (spec 24.5), for its panel, which words it + * in the owner's language. + */ +export type DesktopActionKind = "click" | "type" | "select" | "scroll"; + +export type DesktopAction = { kind: DesktopActionKind, +/** + * The control's name as the window reads it; empty when it has none. + */ +target: string, +/** + * The option chosen, for `select`. + */ +option: string | null, }; + /** * What a bot does on the owner's desktop, for its panel (spec 24.9): the * params of `desktop.changed`. @@ -888,10 +904,9 @@ export type DesktopState = { botId: BotId, */ window: DesktopWindow | null, /** - * What it did last, in English, as its tool said ("Clicked button - * \"Save\"."); `null` after only reading. + * What it did last; `null` after only reading. */ -action: string | null, +action: DesktopAction | null, /** * When it last read or acted, Unix ms. */ diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 29cc7b97..32f88849 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -119,6 +119,29 @@ pub struct DesktopWindow { pub app: String, } +/// What a bot did in a window (spec 24.5), for its panel, which words it +/// in the owner's language. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub enum DesktopActionKind { + Click, + Type, + Select, + Scroll, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopAction { + pub kind: DesktopActionKind, + /// The control's name as the window reads it; empty when it has none. + pub target: String, + /// The option chosen, for `select`. + pub option: Option, +} + /// What a bot does on the owner's desktop, for its panel (spec 24.9): the /// params of `desktop.changed`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -128,9 +151,8 @@ pub struct DesktopState { pub bot_id: BotId, /// The window it read or acted in last. pub window: Option, - /// What it did last, in English, as its tool said ("Clicked button - /// \"Save\"."); `null` after only reading. - pub action: Option, + /// What it did last; `null` after only reading. + pub action: Option, /// When it last read or acted, Unix ms. pub at: Option, /// The owner stopped it (spec 24.9): its desktop tools refuse until diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index 131f090b..0230b6ca 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -13,6 +13,8 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs index 67bad06c..3c5d111a 100644 --- a/crates/botloftd/src/desktop/activity.rs +++ b/crates/botloftd/src/desktop/activity.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use std::sync::{Mutex, PoisonError}; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopState, DesktopWindow}; +use botloft_core::protocol::{DesktopAction, DesktopState, DesktopWindow}; use crate::platform::desktop::Window; @@ -57,7 +57,7 @@ impl Activity { &self, bot: &BotId, window: &Window, - action: Option, + action: Option, now: i64, ) -> DesktopState { self.change(bot, |state| { @@ -104,12 +104,12 @@ mod tests { minimized: false, elevated: false, }; - let used = activity.used( - &bot, - &window, - Some("Clicked button \"Save\".".to_owned()), - 9, - ); + let click = DesktopAction { + kind: botloft_core::protocol::DesktopActionKind::Click, + target: "Save".to_owned(), + option: None, + }; + let used = activity.used(&bot, &window, Some(click), 9); assert_eq!(activity.window(&bot), Some(7)); assert_eq!( used.window.map(|window| window.app), diff --git a/crates/botloftd/src/platform/desktop/windows/act.rs b/crates/botloftd/src/platform/desktop/windows/act.rs index bbff3cb8..1928edb5 100644 --- a/crates/botloftd/src/platform/desktop/windows/act.rs +++ b/crates/botloftd/src/platform/desktop/windows/act.rs @@ -4,23 +4,20 @@ use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - ExpandCollapseState_Collapsed, IUIAutomation2, IUIAutomationElement, - IUIAutomationExpandCollapsePattern, IUIAutomationInvokePattern, - IUIAutomationLegacyIAccessiblePattern, IUIAutomationScrollItemPattern, - IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, IUIAutomationTogglePattern, - IUIAutomationValuePattern, ScrollAmount_LargeDecrement, ScrollAmount_LargeIncrement, - ScrollAmount_NoAmount, TreeScope_Descendants, UIA_ExpandCollapsePatternId, UIA_InvokePatternId, - UIA_LegacyIAccessiblePatternId, UIA_NamePropertyId, UIA_ScrollItemPatternId, - UIA_ScrollPatternId, UIA_SelectionItemPatternId, UIA_TogglePatternId, UIA_ValuePatternId, + ExpandCollapseState_Collapsed, IUIAutomationElement, IUIAutomationExpandCollapsePattern, + IUIAutomationInvokePattern, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationScrollItemPattern, IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, + IUIAutomationTogglePattern, IUIAutomationValuePattern, ScrollAmount_LargeDecrement, + ScrollAmount_LargeIncrement, ScrollAmount_NoAmount, TreeScope_Descendants, + UIA_ExpandCollapsePatternId, UIA_InvokePatternId, UIA_LegacyIAccessiblePatternId, + UIA_NamePropertyId, UIA_ScrollItemPatternId, UIA_ScrollPatternId, UIA_SelectionItemPatternId, + UIA_TogglePatternId, UIA_ValuePatternId, }; use windows::core::{BSTR, Interface}; use super::super::{Acted, Action, DesktopError, Scroll}; use super::uia::{Session, system}; -/// How long a call into the app may take: one that opens a dialog may not -/// come back until the dialog closes. -const TRANSACTION_MS: u32 = 5000; /// UI Automation's "the app took too long". const TIMEOUT: windows::core::HRESULT = windows::core::HRESULT(0x8013_1505_u32 as i32); /// "No scroll" for one direction of `SetScrollPercent`. @@ -28,10 +25,6 @@ const NO_SCROLL: f64 = -1.0; pub fn act(id: u64, target: &[i32], action: &Action) -> Result { let session = Session::start()?; - // SAFETY: a COM call on this session's interface. - if let Ok(quick) = session.automation.cast::() { - let _ = unsafe { quick.SetTransactionTimeout(TRANSACTION_MS) }; - } let (element, control) = session.find(id, target)?; if control.password { return Err(DesktopError::Password); diff --git a/crates/botloftd/src/platform/desktop/windows/act_tests.rs b/crates/botloftd/src/platform/desktop/windows/act_tests.rs index 31026e1a..a579ba6e 100644 --- a/crates/botloftd/src/platform/desktop/windows/act_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/act_tests.rs @@ -22,6 +22,17 @@ fn with_value(window: &TestWindow, value: &str) -> Control { .unwrap_or_else(|| panic!("no control with the value {value:?}")) } +/// Whether `done` comes true within two seconds. +fn soon(done: impl Fn() -> bool) -> bool { + for _ in 0..20 { + if done() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + done() +} + fn title(window: &TestWindow) -> String { listed() .expect("windows") @@ -84,12 +95,19 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .expect("the combo box"); let large = Action::Select("Large".to_owned()); act(window.id, &size.runtime_id, &large).expect("choose"); - let chosen = read(window.id) - .expect("read") - .into_iter() - .find(|control| control.kind == "combo box") - .expect("the combo box"); - assert_eq!(chosen.value.as_deref(), Some("Large")); + // The window takes the choice a moment later, as the tool waits for. + let chosen = || { + read(window.id) + .ok()? + .into_iter() + .find(|control| control.kind == "combo box") + .and_then(|control| control.value) + }; + assert!( + soon(|| chosen().as_deref() == Some("Large")), + "{:?}", + chosen() + ); let none = Action::Select("Huge".to_owned()); assert!(matches!( act(window.id, &size.runtime_id, &none), @@ -110,7 +128,7 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .find(|control| control.kind == "list") .expect("the list"); act(window.id, &list.runtime_id, &Action::Scroll(Scroll::Bottom)).expect("scroll"); - assert!(shown(&last), "scrolled to its end"); + assert!(soon(|| shown(&last)), "scrolled to its end"); } #[test] diff --git a/crates/botloftd/src/platform/desktop/windows/uia.rs b/crates/botloftd/src/platform/desktop/windows/uia.rs index 03519b26..08ebb59e 100644 --- a/crates/botloftd/src/platform/desktop/windows/uia.rs +++ b/crates/botloftd/src/platform/desktop/windows/uia.rs @@ -5,14 +5,15 @@ //! control a reading named, to act on it. use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; use windows::Win32::System::Com::{ CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, }; use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationCondition, - IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, + CUIAutomation, IUIAutomation, IUIAutomation2, IUIAutomationCacheRequest, + IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_IsPasswordPropertyId, UIA_IsValuePatternAvailablePropertyId, UIA_NamePropertyId, UIA_PROPERTY_ID, UIA_RuntimeIdPropertyId, @@ -24,11 +25,16 @@ use super::super::DesktopError; use super::super::read::Control; use super::handle; use super::uia_control::{control, kind_of}; +use windows::core::Interface; -/// The most controls one walk gathers, and how deep it goes: a huge -/// window is cut short, not read for minutes. +/// The most controls one walk gathers, how deep it goes and how long it +/// takes: a huge or stuck window is cut short, not read for minutes. const CONTROLS_MAX: usize = 3000; const DEPTH_MAX: usize = 40; +const WALK_MAX: Duration = Duration::from_secs(15); +/// How long one call into an app may take, and connecting to it: an app +/// busy with a dialog, or stuck, would otherwise hold the call for long. +const CALL_MS: u32 = 5000; const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ UIA_NamePropertyId, @@ -95,6 +101,10 @@ impl Session { let automation: IUIAutomation = CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) .map_err(|err| system(&err))?; + if let Ok(limited) = automation.cast::() { + let _ = limited.SetTransactionTimeout(CALL_MS); + let _ = limited.SetConnectionTimeout(CALL_MS); + } let cache = automation .CreateCacheRequest() .map_err(|err| system(&err))?; @@ -138,6 +148,7 @@ impl Session { target, hit: None, found: Vec::new(), + until: Instant::now() + WALK_MAX, }; // SAFETY: as above. unsafe { walk.children(&root, 0) }; @@ -167,6 +178,8 @@ struct Walk<'a> { target: Option<&'a [i32]>, hit: Option, found: Vec, + /// When the walk gives up and keeps what it has. + until: Instant, } /// The controls of window `id`, in the order they sit in it. @@ -177,7 +190,11 @@ pub fn read(id: u64) -> Result, DesktopError> { impl Walk<'_> { /// Adds the shown children of `element`, and theirs. unsafe fn children(&mut self, element: &IUIAutomationElement, depth: usize) { - if depth > DEPTH_MAX || self.found.len() >= CONTROLS_MAX || self.hit.is_some() { + if depth > DEPTH_MAX + || self.found.len() >= CONTROLS_MAX + || self.hit.is_some() + || Instant::now() > self.until + { return; } let session = self.session; diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index 08464846..62b04788 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -7,7 +7,7 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use botloft_core::ids::BotId; -use botloft_core::protocol::DesktopLevel; +use botloft_core::protocol::{DesktopAction, DesktopLevel}; use serde_json::{Value, json}; use tracing::debug; @@ -216,7 +216,7 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul } /// The bot read or acted in `window`: its panel shows it (spec 24.9). -pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { let now = daemon.clock.now_ms(); let state = daemon.desktop.activity.used(bot, window, action, now); daemon.emit(Event::DesktopChanged(state)); diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index cf1cec8c..ed43eab9 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -6,7 +6,7 @@ use std::time::Duration; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use botloft_core::protocol::{DesktopAction, DesktopActionKind, DesktopGrant, DesktopLevel}; use super::desktop::used; use super::desktop_grant::{blocking, describe, granted, heading}; @@ -40,6 +40,21 @@ fn done(action: &Action, control: &Control) -> String { } } +/// The action as the bot's panel shows it. +fn shown(action: &Action, control: &Control) -> DesktopAction { + let (kind, option) = match action { + Action::Click => (DesktopActionKind::Click, None), + Action::Type(_) => (DesktopActionKind::Type, None), + Action::Select(option) => (DesktopActionKind::Select, Some(option.clone())), + Action::Scroll(_) => (DesktopActionKind::Scroll, None), + }; + DesktopAction { + kind, + target: control.name.clone(), + option, + } +} + pub(super) async fn act( daemon: &Daemon, bot: &BotId, @@ -92,7 +107,7 @@ pub(super) async fn act( let (text, next) = render(&controls, 0); daemon.desktop.keep(bot, id, controls); let mut answer = done(&ask.action, &control); - used(daemon, bot, &window, Some(answer.clone())); + used(daemon, bot, &window, Some(shown(&ask.action, &control))); if acted == Acted::Waiting { answer.push_str( " The app did not answer in time: it may be showing a dialog. Call desktop_windows \ diff --git a/docs/spec.md b/docs/spec.md index 2c64c4a2..273c2c37 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1713,7 +1713,7 @@ O bot lê e usa as janelas pela **UI Automation** do Windows (`IUIAutomation`, C - escolher numa lista: o item pelo nome (`FindFirst` nos descendentes) com `SelectionItem.Select`, quando ele aparece sem abrir nada. Numa caixa de opções, a lista só existe aberta, e a clássica do Win32 só aceita a escolha pela ação padrão do item na lista aberta (`LegacyIAccessible.DoDefaultAction`, um clique duplo), que também a fecha; fechá-la de outro jeito cancela a escolha, como Esc (visto no Windows 11). A lista aberta fecha se a janela perde o foco no meio: aí a tool diz que não achou a opção, e o bot tenta de novo; - rolar: `Scroll` (uma página para baixo ou para cima, ou `SetScrollPercent` ao topo ou ao fim), ou, num item de lista, `ScrollItem.ScrollIntoView`. - Antes de agir, o controle é achado de novo pelo `RuntimeId` guardado na leitura, numa descida que para ao achá-lo; sumiu, a tool diz para ler de novo. Controle desativado não é usado. -- Cada chamada a um app tem 5 s (`IUIAutomation2.TransactionTimeout`): um clique que abre um diálogo pode só voltar quando ele fecha. Passado esse tempo, a ação vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. +- Cada chamada a um app tem 5 s, e conectar-se a ele também (`IUIAutomation2.TransactionTimeout` e `ConnectionTimeout`, em toda sessão, de leitura também): um clique que abre um diálogo pode só voltar quando ele fecha, e um app ocupado ou travado seguraria a chamada por muito tempo. Passado esse tempo numa ação, ela vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. A leitura de uma janela inteira para em 15 s e fica com o que leu. Visto no Windows 11: logo depois de uma caixa de opções escolhida pela ação padrão, a janela pode demorar a responder; sem esses limites, a leitura seguinte ficou presa por minutos. - Uma sessão de UI Automation por vez no processo do daemon: duas, em threads diferentes ao mesmo tempo, falham com "erro não especificado" (visto no Windows 11). - Sem o padrão que a ação pede, a tool explica o que faltou. Com mouse e teclado de verdade ligados (24.7), ela faz a ação com eles; sem, diz que precisaria deles. - Depois de agir, a tool espera 400 ms e devolve a janela lida de novo, com o título de agora (um clique pode mudá-lo), como o navegador. Ela abre com o que fez: `Clicked button "Save".`, `Typed in edit.`, `Chose "Large" in combo box.`, `Scrolled list.`. @@ -1756,7 +1756,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App - **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. -- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação como a tool disse (`Clicked button "Save".`; nada depois de só ler), quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. - **Parar tudo:** o atalho global `Ctrl+Alt+End`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele. From 978bd0575d4ec54c4d27f78d32777764179be4cd Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:50:01 -0300 Subject: [PATCH 3/3] feat(app): the desktop panel, live, with Stop The app part of D3 (spec 24.9): the desktop is the fourth place of a bot's computer dock, beside its browser, terminal and files, at the same width. The panel watches the bot's desktop while open: the window it read or acted in last, live on the bot's desk, the app and the title, and what it did, worded in the owner's language from the structured action. Stop in the header; stopped, a badge and a notice with Let it go on. Without a window yet it says when one shows; at its foot, the Ctrl+Alt+End hint. Desktop tool lines in the chat open it. The preview shows Scout's Notepad. --- app/src/dev/seedDesktop.ts | 34 +++++- app/src/features/bots/BotView.tsx | 17 ++- app/src/features/chat/ToolLines.tsx | 13 +++ app/src/features/desktop/DesktopPanel.tsx | 128 +++++++++++++++++++++ app/src/features/desktop/panel.test.tsx | 71 ++++++++++++ app/src/features/desktop/showDesktop.ts | 11 ++ app/src/features/desktop/useDesktopView.ts | 51 ++++++++ app/src/features/terminal/ComputerDock.tsx | 16 +-- app/src/i18n/en/desktop.ts | 26 +++++ app/src/i18n/en/terminal.ts | 1 + app/src/i18n/es/desktop.ts | 28 +++++ app/src/i18n/es/terminal.ts | 1 + app/src/i18n/pt-BR/desktop.ts | 27 +++++ app/src/i18n/pt-BR/terminal.ts | 1 + app/src/store/app.ts | 2 +- docs/spec.md | 4 +- 16 files changed, 418 insertions(+), 13 deletions(-) create mode 100644 app/src/features/desktop/DesktopPanel.tsx create mode 100644 app/src/features/desktop/panel.test.tsx create mode 100644 app/src/features/desktop/showDesktop.ts create mode 100644 app/src/features/desktop/useDesktopView.ts diff --git a/app/src/dev/seedDesktop.ts b/app/src/dev/seedDesktop.ts index 6b654988..0ce7e2a5 100644 --- a/app/src/dev/seedDesktop.ts +++ b/app/src/dev/seedDesktop.ts @@ -1,13 +1,45 @@ // Dev only: the preview's desktop (spec 24). Analyst asks to see a -// spreadsheet in Excel, and Scout may already see Notepad. +// spreadsheet in Excel, and Scout may already see Notepad, where it just +// clicked Save. import type { FakeBotloft } from "../lib/fake"; import type { BotId } from "../lib/protocol.gen"; const EXCEL = "C:\\Program Files\\Microsoft Office\\root\\Office16\\EXCEL.EXE"; +const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" }; + +/** A Notepad window, drawn: a title bar, a menu and a few lines. */ +function drawNotepad(): string { + const canvas = document.createElement("canvas"); + canvas.width = 760; + canvas.height = 480; + const g = canvas.getContext("2d"); + if (!g) { + return ""; + } + g.fillStyle = "#ffffff"; + g.fillRect(0, 0, 760, 480); + g.fillStyle = "#f3f3f3"; + g.fillRect(0, 0, 760, 72); + g.fillStyle = "#1f1f1f"; + g.font = "14px system-ui, sans-serif"; + g.fillText(NOTES.title, 16, 24); + g.fillText("File Edit View", 16, 58); + g.font = "16px Consolas, monospace"; + ["Flour, 2 kg", "Eggs, 12", "Butter, 500 g", "Sugar, 1 kg"].forEach((line, index) => { + g.fillText(line, 20, 112 + index * 26); + }); + return canvas.toDataURL("image/jpeg", 0.8).split(",")[1] ?? ""; +} export function seedDesktop(fake: FakeBotloft, bots: { scout: BotId; analyst: BotId }) { fake.desktop.grant(bots.scout, "C:\\Windows\\notepad.exe", "Notepad"); + fake.desktop.use(bots.scout, NOTES, { kind: "click", target: "Save", option: null }); + fake.desktop.paint(bots.scout, drawNotepad(), 760, 480); + fake.chat.tool(bots.scout, "mcp__botloft__desktop_click", { + summary: "To save the list", + status: "done", + }); fake.chat.ask( bots.analyst, "mcp__botloft__desktop", diff --git a/app/src/features/bots/BotView.tsx b/app/src/features/bots/BotView.tsx index d373ec5f..83559fdf 100644 --- a/app/src/features/bots/BotView.tsx +++ b/app/src/features/bots/BotView.tsx @@ -11,6 +11,8 @@ import { useStable } from "../../ui/useStable"; import { BrowserPanel } from "../browser/BrowserPanel"; import { ShowBrowser } from "../browser/showBrowser"; import { ChatView } from "../chat/ChatView"; +import { DesktopPanel } from "../desktop/DesktopPanel"; +import { ShowDesktop } from "../desktop/showDesktop"; import { FilesPanel } from "../files/FilesPanel"; import { ShowFile } from "../files/showFile"; import { useBotFiles } from "../files/useBotFiles"; @@ -134,14 +136,22 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { } setSide("terminal"); }); + const showDesktop = useStable(() => { + if (filesOpen) { + seen(); + } + setSide("desktop"); + }); // The dock of the bot's computer: browser, terminal and files. const dock = { open: side, - pick: (place: "browser" | "terminal" | "files") => { + pick: (place: "browser" | "terminal" | "files" | "desktop") => { if (place === "browser") { showBrowser(); } else if (place === "terminal") { showTerminal(); + } else if (place === "desktop") { + showDesktop(); } else if (!filesOpen) { toggleFiles(); } @@ -184,7 +194,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { - + + + @@ -221,6 +233,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) { /> )} {beside === "terminal" && setSide(null)} />} + {beside === "desktop" && setSide(null)} />} diff --git a/app/src/features/chat/ToolLines.tsx b/app/src/features/chat/ToolLines.tsx index 1a4bf4cb..7610d75a 100644 --- a/app/src/features/chat/ToolLines.tsx +++ b/app/src/features/chat/ToolLines.tsx @@ -30,6 +30,7 @@ import { useT } from "../../i18n"; import type { ChatItem, ToolItem } from "../../lib/protocol.gen"; import { useArrival } from "../../ui/motion"; import { isBrowserTool, ShowBrowser } from "../browser/showBrowser"; +import { isDesktopTool, ShowDesktop } from "../desktop/showDesktop"; import { ShowFile } from "../files/showFile"; import { isScreenFile, ShowScreen } from "../screens/showScreen"; import { ShowTerminal } from "../terminal/showTerminal"; @@ -106,6 +107,7 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) { const showBrowser = useContext(ShowBrowser); const showScreen = useContext(ShowScreen); const showTerminal = useContext(ShowTerminal); + const showDesktop = useContext(ShowDesktop); const arrival = useArrival(createdAt); const title = toolTitle(tool.name, t.tools); // A command reads in the bot's own words, when it gave them; the command @@ -173,6 +175,17 @@ function ToolLine({ tool, createdAt }: { tool: ToolItem; createdAt: number }) { )} + {showDesktop && isDesktopTool(tool.name) && ( + + )} {showBrowser && isBrowserTool(tool.name) && ( + )} + + + + )} + {state && !window && ( + + )} + {window && ( + <> +

+ {window.app} + + {window.title} + +

+
+ {frame ? ( + {window.title} + ) : ( +

{t.waiting}

+ )} +
+

+ + {actionText(state?.action ?? null, t)} +

+ + )} +

{t.shortcut}

+ + + + ); +} diff --git a/app/src/features/desktop/panel.test.tsx b/app/src/features/desktop/panel.test.tsx new file mode 100644 index 00000000..75e86eed --- /dev/null +++ b/app/src/features/desktop/panel.test.tsx @@ -0,0 +1,71 @@ +import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import { afterEach, describe, expect, test } from "vitest"; +import { FakeBotloft } from "../../lib/fake"; +import { crewOpened, openBot, renderApp } from "../../test/app"; + +afterEach(cleanup); + +async function openScout() { + const fake = new FakeBotloft(); + const ops = fake.addCrew("Ops"); + const scout = fake.addBot(ops.id, "Scout", "Finds sources"); + fake.setBotState(scout.id, "idle"); + fake.chat.tool(scout.id, "mcp__botloft__desktop_look", { + summary: "To read the list", + status: "done", + }); + renderApp(fake); + await crewOpened("Ops"); + openBot("Scout"); + await screen.findByRole("list", { name: "Messages" }); + return { fake, scout }; +} + +const panel = () => screen.getByRole("complementary", { name: "Scout's desktop" }); +const NOTES = { id: 42, title: "Shopping list - Notepad", app: "Notepad" }; + +describe("the desktop panel", () => { + test("opens from a desktop tool in the chat and shows the window the bot uses, live", async () => { + const { fake, scout } = await openScout(); + fireEvent.click( + screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }), + ); + expect(await within(panel()).findByText("Scout has not used your desktop yet")).toBeDefined(); + expect(fake.desktop.watching).toBe(scout.id); + + act(() => { + fake.desktop.use(scout.id, NOTES, { kind: "click", target: "Save", option: null }); + }); + expect(await within(panel()).findByText("Shopping list - Notepad")).toBeDefined(); + expect(within(panel()).getByText('Clicked "Save"')).toBeDefined(); + expect(within(panel()).getByText("Waiting for the picture…")).toBeDefined(); + + act(() => { + fake.desktop.paint(scout.id, "AAAA"); + }); + const picture = await within(panel()).findByRole("img", { name: "Shopping list - Notepad" }); + expect(picture.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA"); + expect(within(panel()).getByText("Live")).toBeDefined(); + + // Closed, nobody watches. + fireEvent.click(within(panel()).getByRole("button", { name: "Close" })); + await waitFor(() => expect(fake.desktop.watching).toBeNull()); + }); + + test("stops the bot on the desktop and lets it go on", async () => { + const { fake, scout } = await openScout(); + fireEvent.click( + screen.getByRole("button", { name: "Watch on the desktop panel: Read a window" }), + ); + act(() => { + fake.desktop.use(scout.id, NOTES); + }); + expect(await within(panel()).findByText("Read the window")).toBeDefined(); + fireEvent.click(within(panel()).getByRole("button", { name: "Stop" })); + expect(await within(panel()).findByText("You stopped Scout on your desktop")).toBeDefined(); + expect(fake.desktop.state(scout.id).stopped).toBe(true); + fireEvent.click(within(panel()).getByRole("button", { name: "Let it go on" })); + await waitFor(() => expect(fake.desktop.state(scout.id).stopped).toBe(false)); + expect(within(panel()).queryByText("You stopped Scout on your desktop")).toBeNull(); + }); +}); diff --git a/app/src/features/desktop/showDesktop.ts b/app/src/features/desktop/showDesktop.ts new file mode 100644 index 00000000..3e6bb17d --- /dev/null +++ b/app/src/features/desktop/showDesktop.ts @@ -0,0 +1,11 @@ +// How a desktop tool line in the chat opens the bot's desktop panel (spec +// 24.9). Absent where there is no panel, so no button shows. + +import { createContext } from "react"; + +export const ShowDesktop = createContext<(() => void) | null>(null); + +/** `mcp__botloft__desktop_look` and the other desktop tools. */ +export function isDesktopTool(name: string): boolean { + return name.startsWith("mcp__botloft__desktop_"); +} diff --git a/app/src/features/desktop/useDesktopView.ts b/app/src/features/desktop/useDesktopView.ts new file mode 100644 index 00000000..2c585f72 --- /dev/null +++ b/app/src/features/desktop/useDesktopView.ts @@ -0,0 +1,51 @@ +// What a bot does on the owner's desktop, live (spec 24.9): its state and +// the picture of the window it is using, while its panel is open. + +import { useEffect, useState } from "react"; +import type { BotId, DesktopFrame, DesktopState } from "../../lib/protocol.gen"; +import { useApi } from "../../store/context"; + +export interface DesktopView { + /** `null` until the daemon answers. */ + state: DesktopState | null; + frame: DesktopFrame | null; +} + +export function useDesktopView(botId: BotId): DesktopView { + const api = useApi(); + const [view, setView] = useState({ state: null, frame: null }); + useEffect(() => { + let alive = true; + setView({ state: null, frame: null }); + const stop = api.subscribe((event) => { + if (event.name === "desktop.changed" && event.params.botId === botId) { + setView((now) => ({ ...now, state: event.params })); + } + if (event.name === "desktop.frame" && event.params.botId === botId) { + setView((now) => ({ ...now, frame: event.params })); + } + }); + api.call("desktop.watch", { botId }).then( + // What arrived meanwhile is newer than the answer. + (watched) => + alive && + setView((now) => ({ + state: now.state ?? watched.state, + frame: now.frame ?? watched.frame, + })), + // An older daemon has no desktop: nothing to show. + () => + alive && + setView({ + state: { botId, window: null, action: null, at: null, stopped: false }, + frame: null, + }), + ); + return () => { + alive = false; + stop(); + api.call("desktop.unwatch").catch(() => {}); + }; + }, [api, botId]); + return view; +} diff --git a/app/src/features/terminal/ComputerDock.tsx b/app/src/features/terminal/ComputerDock.tsx index ea8f3172..c65bfa9d 100644 --- a/app/src/features/terminal/ComputerDock.tsx +++ b/app/src/features/terminal/ComputerDock.tsx @@ -1,14 +1,14 @@ -// The dock of a bot's computer (spec 15.1): its browser, its terminal and -// its files, one click apart, at the foot of each of those panels. The -// three share one width (`computer`), so moving between them only changes -// what is inside. +// The dock of a bot's computer (spec 15.1): its browser, its terminal, its +// files and the owner's desktop as the bot uses it (spec 24.9), one click +// apart, at the foot of each of those panels. They share one width +// (`computer`), so moving between them only changes what is inside. -import { Folder, Globe, type LucideIcon, SquareTerminal } from "lucide-react"; +import { Folder, Globe, type LucideIcon, Monitor, SquareTerminal } from "lucide-react"; import { createContext, useContext } from "react"; import { useT } from "../../i18n"; import type { BotPanel } from "../../store/app"; -type Place = Extract; +type Place = Extract; /** The panel open and how to open another; absent outside a bot's view. */ export const Dock = createContext<{ open: BotPanel | null; pick(place: Place): void } | null>(null); @@ -17,11 +17,13 @@ const PLACES: { place: Place; icon: LucideIcon }[] = [ { place: "browser", icon: Globe }, { place: "terminal", icon: SquareTerminal }, { place: "files", icon: Folder }, + { place: "desktop", icon: Monitor }, ]; /** * The dock as the foot of a panel: the same place in the browser, the - * terminal and the files, so it stays still while the panel above changes. + * terminal, the files and the desktop, so it stays still while the panel + * above changes. */ export function ComputerFooter() { const dock = useContext(Dock); diff --git a/app/src/i18n/en/desktop.ts b/app/src/i18n/en/desktop.ts index 2ace916d..dfaf1ec5 100644 --- a/app/src/i18n/en/desktop.ts +++ b/app/src/i18n/en/desktop.ts @@ -18,6 +18,32 @@ export const desktop = { allowedUse: (bot: string, app: string) => `${bot} can use ${app}`, deniedUse: (bot: string, app: string) => `${bot} may not use ${app}`, }, + panel: { + label: (bot: string) => `${bot}'s desktop`, + heading: "Desktop", + live: "Live", + stoppedBadge: "Stopped", + stop: "Stop", + stopFailed: "Could not stop it", + resume: "Let it go on", + resumeFailed: "Could not let it go on", + stoppedTitle: (bot: string) => `You stopped ${bot} on your desktop`, + stoppedBody: (bot: string) => `${bot} cannot read or use your apps until you let it go on.`, + emptyTitle: (bot: string) => `${bot} has not used your desktop yet`, + emptyBody: "When it reads or uses a window of an app you allowed, the window shows here, live.", + shortcut: "Ctrl+Alt+End stops every bot on your desktop, even with Botloft closed.", + waiting: "Waiting for the picture…", + read: "Read the window", + click: (target: string) => `Clicked "${target}"`, + clickSomething: "Clicked a control", + type: (target: string) => `Typed in "${target}"`, + typeSomething: "Typed in a field", + select: (option: string) => `Chose "${option}"`, + scroll: (target: string) => `Scrolled "${target}"`, + scrollSomething: "Scrolled", + showInPanel: "Watch on the desktop panel", + close: "Close", + }, grants: { title: "Your desktop", none: (bot: string) => diff --git a/app/src/i18n/en/terminal.ts b/app/src/i18n/en/terminal.ts index a6e11590..3d1307bd 100644 --- a/app/src/i18n/en/terminal.ts +++ b/app/src/i18n/en/terminal.ts @@ -16,5 +16,6 @@ export const terminal = { browser: "Browser", terminal: "Terminal", files: "Files", + desktop: "Desktop", }, }; diff --git a/app/src/i18n/es/desktop.ts b/app/src/i18n/es/desktop.ts index 835c78a3..e434d568 100644 --- a/app/src/i18n/es/desktop.ts +++ b/app/src/i18n/es/desktop.ts @@ -17,6 +17,34 @@ export const desktop: Messages["desktop"] = { allowedUse: (bot, app) => `${bot} puede usar ${app}`, deniedUse: (bot, app) => `${bot} no puede usar ${app}`, }, + panel: { + label: (bot) => `Escritorio de ${bot}`, + heading: "Escritorio", + live: "En vivo", + stoppedBadge: "Detenido", + stop: "Detener", + stopFailed: "No se pudo detener", + resume: "Dejar que siga", + resumeFailed: "No se pudo dejar que siga", + stoppedTitle: (bot) => `Detuviste a ${bot} en tu escritorio`, + stoppedBody: (bot) => `${bot} no lee ni usa tus apps hasta que lo dejes seguir.`, + emptyTitle: (bot) => `${bot} todavía no usó tu escritorio`, + emptyBody: + "Cuando lea o use una ventana de una app que permitiste, la ventana aparece aquí, en vivo.", + shortcut: + "Ctrl+Alt+Fin detiene a todos los bots en tu escritorio, incluso con Botloft cerrado.", + waiting: "Esperando la imagen…", + read: "Leyó la ventana", + click: (target) => `Hizo clic en "${target}"`, + clickSomething: "Hizo clic en un control", + type: (target) => `Escribió en "${target}"`, + typeSomething: "Escribió en un campo", + select: (option) => `Eligió "${option}"`, + scroll: (target) => `Desplazó "${target}"`, + scrollSomething: "Desplazó", + showInPanel: "Ver en el panel del escritorio", + close: "Cerrar", + }, grants: { title: "Tu escritorio", none: (bot) => diff --git a/app/src/i18n/es/terminal.ts b/app/src/i18n/es/terminal.ts index 1eba798e..72e526ec 100644 --- a/app/src/i18n/es/terminal.ts +++ b/app/src/i18n/es/terminal.ts @@ -14,5 +14,6 @@ export const terminal: Messages["terminal"] = { browser: "Navegador", terminal: "Terminal", files: "Archivos", + desktop: "Escritorio", }, }; diff --git a/app/src/i18n/pt-BR/desktop.ts b/app/src/i18n/pt-BR/desktop.ts index 6819ab87..21e048b4 100644 --- a/app/src/i18n/pt-BR/desktop.ts +++ b/app/src/i18n/pt-BR/desktop.ts @@ -17,6 +17,33 @@ export const desktop: Messages["desktop"] = { allowedUse: (bot, app) => `${bot} pode usar ${app}`, deniedUse: (bot, app) => `${bot} não pode usar ${app}`, }, + panel: { + label: (bot) => `Desktop de ${bot}`, + heading: "Desktop", + live: "Ao vivo", + stoppedBadge: "Parado", + stop: "Parar", + stopFailed: "Não foi possível parar", + resume: "Deixar continuar", + resumeFailed: "Não foi possível deixar continuar", + stoppedTitle: (bot) => `Você parou ${bot} no seu desktop`, + stoppedBody: (bot) => `${bot} não lê nem usa seus apps até você deixar continuar.`, + emptyTitle: (bot) => `${bot} ainda não usou seu desktop`, + emptyBody: + "Quando ele ler ou usar uma janela de um app que você liberou, a janela aparece aqui, ao vivo.", + shortcut: "Ctrl+Alt+End para todos os bots no seu desktop, mesmo com o Botloft fechado.", + waiting: "Esperando a imagem…", + read: "Leu a janela", + click: (target) => `Clicou em "${target}"`, + clickSomething: "Clicou num controle", + type: (target) => `Escreveu em "${target}"`, + typeSomething: "Escreveu num campo", + select: (option) => `Escolheu "${option}"`, + scroll: (target) => `Rolou "${target}"`, + scrollSomething: "Rolou", + showInPanel: "Ver no painel do desktop", + close: "Fechar", + }, grants: { title: "Seu desktop", none: (bot) => diff --git a/app/src/i18n/pt-BR/terminal.ts b/app/src/i18n/pt-BR/terminal.ts index 47c9b57a..963c75fc 100644 --- a/app/src/i18n/pt-BR/terminal.ts +++ b/app/src/i18n/pt-BR/terminal.ts @@ -14,5 +14,6 @@ export const terminal: Messages["terminal"] = { browser: "Navegador", terminal: "Terminal", files: "Arquivos", + desktop: "Desktop", }, }; diff --git a/app/src/store/app.ts b/app/src/store/app.ts index cfd4bf3e..2464358d 100644 --- a/app/src/store/app.ts +++ b/app/src/store/app.ts @@ -28,7 +28,7 @@ import { withQuestion } from "./questions"; import { loadMarked, loadSeen, loadSeenSince, seenActions, withReply } from "./seen"; /** A panel beside a bot's chat (spec 15.1). */ -export type BotPanel = "details" | "files" | "browser" | "screens" | "terminal"; +export type BotPanel = "details" | "files" | "browser" | "screens" | "terminal" | "desktop"; export interface AppState { connection: ConnectionState; diff --git a/docs/spec.md b/docs/spec.md index 273c2c37..c8838eaf 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1653,7 +1653,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 e D2 implementados; D3 em andamento** (24.12): o daemon já guarda o que cada bot faz no desktop, manda a foto ao vivo, para e deixa continuar, e tem o atalho; falta o painel no app. É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1, D2 e D3 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1755,7 +1755,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. +- **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+End. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. - **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado.