From 810eceedbc860ab26e77273b544913eb249783a5 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:24:40 -0300 Subject: [PATCH 1/2] feat(desktop): what each bot does on the desktop, live, and stopping it The daemon part of D3 (spec 24.9): - Each bot's DesktopState, in memory: the window it read or acted in last, its last action as the tool said it, when, and whether the owner stopped it; every reading, picture and action updates it and sends desktop.changed. - desktop.watch and desktop.unwatch: while a connection watches a bot, its window is pictured four times a second and desktop.frame goes out only when the picture changed, only to that connection, newest first, like the browser's frames. The picturing stops with the last watcher. - desktop.stop and desktop.resume: a stopped bot's desktop tools refuse, again after waiting for the owner or for its turn, until the owner lets it go on; an action under way ends within its 5 seconds. - Ctrl+Alt+End, registered by the daemon on a thread with its own queue, stops every bot on the desktop, app open or not; when another program holds it, the daemon logs it and goes on. The window and setup of the desktop tests move to tests/common, and the event-to-notification mapping out of rpc/mod.rs, which had reached 300 lines. FakeBotloft plays the new methods. --- app/src/lib/fakeDesktop.ts | 66 ++++++- app/src/lib/protocol.gen.ts | 59 ++++++ crates/botloft-core/src/protocol/desktop.rs | 60 +++++++ .../src/protocol/export_desktop.rs | 19 ++ crates/botloft-core/src/protocol/mod.rs | 7 + crates/botloftd/src/desktop/activity.rs | 128 +++++++++++++ .../src/{desktop.rs => desktop/mod.rs} | 30 +++- crates/botloftd/src/desktop/screen.rs | 100 +++++++++++ crates/botloftd/src/main.rs | 8 +- crates/botloftd/src/platform/desktop/mod.rs | 14 ++ .../src/platform/desktop/windows/mod.rs | 2 + .../src/platform/desktop/windows/stop_key.rs | 52 ++++++ crates/botloftd/src/rpc/desktop_watch.rs | 50 ++++++ crates/botloftd/src/rpc/dispatch.rs | 2 + crates/botloftd/src/rpc/mod.rs | 48 ++--- crates/botloftd/src/rpc/notify.rs | 34 ++++ crates/botloftd/src/service/desktop.rs | 46 ++++- crates/botloftd/src/state.rs | 4 +- crates/botloftd/src/tools/desktop.rs | 22 ++- crates/botloftd/src/tools/desktop_act.rs | 7 + crates/botloftd/tests/common/desktop.rs | 169 ++++++++++++++++++ crates/botloftd/tests/common/mod.rs | 2 + crates/botloftd/tests/desktop.rs | 165 +---------------- crates/botloftd/tests/desktop_panel.rs | 83 +++++++++ docs/spec.md | 11 +- 25 files changed, 978 insertions(+), 210 deletions(-) create mode 100644 crates/botloftd/src/desktop/activity.rs rename crates/botloftd/src/{desktop.rs => desktop/mod.rs} (83%) create mode 100644 crates/botloftd/src/desktop/screen.rs create mode 100644 crates/botloftd/src/platform/desktop/windows/stop_key.rs create mode 100644 crates/botloftd/src/rpc/desktop_watch.rs create mode 100644 crates/botloftd/src/rpc/notify.rs create mode 100644 crates/botloftd/tests/common/desktop.rs create mode 100644 crates/botloftd/tests/desktop_panel.rs diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index 4483ea44..fabd6cbc 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -1,16 +1,28 @@ -// The fake daemon's desktop grants (spec 24.2, 24.10): what each bot may -// see and do on the owner's desktop, and `desktop.*`. The bots' desktop -// tools are not played: grants come from `grant`, as the real daemon makes -// them when the owner allows a request. +// The fake daemon's desktop (spec 24): what each bot may see and do on the +// owner's desktop (spec 24.2, 24.10), what it is doing there for its panel, +// with pictures while the panel watches, and stopping it (spec 24.9). The +// bots' desktop tools are not played: grants come from `grant` and use +// from `use`, as the real daemon makes them. import type { FakeBotloft, Handlers } from "./fake"; import { notFound } from "./fakeRules"; -import type { BotId, DesktopGrant, DesktopLevel } from "./protocol.gen"; +import type { + BotId, + DesktopFrame, + DesktopGrant, + DesktopLevel, + DesktopState, + DesktopWindow, +} from "./protocol.gen"; type DesktopMethods = Extract; export class FakeDesktop { readonly grants: DesktopGrant[] = []; + private readonly states = new Map(); + private readonly frames = new Map(); + /** The bot whose panel the app watches, if any. */ + watching: BotId | null = null; constructor(private readonly fake: FakeBotloft) {} @@ -48,6 +60,33 @@ export class FakeDesktop { this.fake.emit({ name: "bot.desktop", params: { botId, grants: this.of(botId) } }); } + state(botId: BotId): DesktopState { + return ( + this.states.get(botId) ?? { botId, window: null, action: null, at: null, stopped: false } + ); + } + + private set(botId: BotId, change: Partial): DesktopState { + const state = { ...this.state(botId), ...change }; + this.states.set(botId, state); + this.fake.emit({ name: "desktop.changed", params: state }); + return state; + } + + /** The bot read or acted in `window`; `action` says what it did. */ + use(botId: BotId, window: DesktopWindow, action: string | null = null): DesktopState { + return this.set(botId, { window, action, at: this.fake.now }); + } + + /** A new picture of the bot's window: sent while its panel watches. */ + paint(botId: BotId, data: string, width = 800, height = 600): void { + const frame = { botId, data, width, height }; + this.frames.set(botId, frame); + if (this.watching === botId) { + this.fake.emit({ name: "desktop.frame", params: frame }); + } + } + handlers(): Pick { return { "desktop.grants": ({ botId }) => { @@ -64,6 +103,23 @@ export class FakeDesktop { this.changed(grant.botId); return { botId: grant.botId, grants: this.of(grant.botId) }; }, + "desktop.watch": ({ botId }) => { + this.fake.bot(botId); + this.watching = botId; + return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; + }, + "desktop.unwatch": () => { + this.watching = null; + return null; + }, + "desktop.stop": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: true }); + }, + "desktop.resume": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: false }); + }, }; } } diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 739d0b57..a5c29847 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -869,6 +869,59 @@ export type BotDesktop = { botId: BotId, */ grants: Array, }; +/** + * The window a bot is using on the owner's desktop. + */ +export type DesktopWindow = { id: number, title: string, +/** + * The app's name for people. + */ +app: string, }; + +/** + * What a bot does on the owner's desktop, for its panel (spec 24.9): the + * params of `desktop.changed`. + */ +export type DesktopState = { botId: BotId, +/** + * The window it read or acted in last. + */ +window: DesktopWindow | null, +/** + * What it did last, in English, as its tool said ("Clicked button + * \"Save\"."); `null` after only reading. + */ +action: string | null, +/** + * When it last read or acted, Unix ms. + */ +at: number | null, +/** + * The owner stopped it (spec 24.9): its desktop tools refuse until + * they let it go on. + */ +stopped: boolean, }; + +/** + * A picture of the window a bot is using, for whoever watches its panel. + */ +export type DesktopFrame = { botId: BotId, +/** + * JPEG, base64. + */ +data: string, width: number, height: number, }; + +/** + * The result of `desktop.watch`: the bot's state and, when it uses a + * window, its picture now. + */ +export type DesktopView = { state: DesktopState, frame: DesktopFrame | null, }; + +/** + * `desktop.watch`, `desktop.stop` and `desktop.resume`. + */ +export type DesktopBotParams = { botId: BotId, }; + export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, /** * What the bot says the command is for, as in `ToolItem`. The bot @@ -1551,6 +1604,10 @@ export interface RpcMethods { "rules.delete": { params: RuleIdParams; result: BotRules }; "desktop.grants": { params: DesktopGrantsParams; result: Array }; "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop }; + "desktop.watch": { params: DesktopBotParams; result: DesktopView }; + "desktop.unwatch": { params: undefined; result: null }; + "desktop.stop": { params: DesktopBotParams; result: DesktopState }; + "desktop.resume": { params: DesktopBotParams; result: DesktopState }; "routines.list": { params: RoutinesListParams; result: Array }; "routines.create": { params: RoutinesCreateParams; result: Routine }; "routines.update": { params: RoutinesUpdateParams; result: Routine }; @@ -1600,6 +1657,8 @@ export interface RpcNotifications { "reaction.changed": ReactionChanged; "bot.rules": BotRules; "bot.desktop": BotDesktop; + "desktop.changed": DesktopState; + "desktop.frame": DesktopFrame; "browser.changed": BrowserState; "browser.action": BrowserAction; "browser.frame": BrowserFrame; diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 8be5f5a9..29cc7b97 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -107,3 +107,63 @@ pub struct BotDesktop { /// Oldest first. pub grants: Vec, } + +/// The window a bot is using on the owner's desktop. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopWindow { + pub id: u64, + pub title: String, + /// The app's name for people. + pub app: String, +} + +/// What a bot does on the owner's desktop, for its panel (spec 24.9): the +/// params of `desktop.changed`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopState { + pub bot_id: BotId, + /// The window it read or acted in last. + pub window: Option, + /// What it did last, in English, as its tool said ("Clicked button + /// \"Save\"."); `null` after only reading. + pub action: Option, + /// When it last read or acted, Unix ms. + pub at: Option, + /// The owner stopped it (spec 24.9): its desktop tools refuse until + /// they let it go on. + pub stopped: bool, +} + +/// A picture of the window a bot is using, for whoever watches its panel. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopFrame { + pub bot_id: BotId, + /// JPEG, base64. + pub data: String, + pub width: u32, + pub height: u32, +} + +/// The result of `desktop.watch`: the bot's state and, when it uses a +/// window, its picture now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopView { + pub state: DesktopState, + pub frame: Option, +} + +/// `desktop.watch`, `desktop.stop` and `desktop.resume`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopBotParams { + pub bot_id: BotId, +} diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index 79cd1ef1..131f090b 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -12,6 +12,11 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); } pub(super) fn methods(out: &mut Out) { @@ -25,9 +30,23 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), &out.name::(), ); + let bot = out.name::(); + out.method(method::DESKTOP_WATCH, &bot, &out.name::()); + out.method(method::DESKTOP_UNWATCH, "undefined", "null"); + let state = out.name::(); + out.method(method::DESKTOP_STOP, &bot, &state); + out.method(method::DESKTOP_RESUME, &bot, &state); } pub(super) fn notifications(out: &mut Out) { let desktop = out.name::(); let _ = writeln!(out.text, " \"{}\": {desktop};", notification::BOT_DESKTOP); + let state = out.name::(); + let _ = writeln!( + out.text, + " \"{}\": {state};", + notification::DESKTOP_CHANGED + ); + let frame = out.name::(); + let _ = writeln!(out.text, " \"{}\": {frame};", notification::DESKTOP_FRAME); } diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index f1a7ecb9..cac4565e 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -125,6 +125,10 @@ pub mod method { pub const RULES_DELETE: &str = "rules.delete"; pub const DESKTOP_GRANTS: &str = "desktop.grants"; pub const DESKTOP_REVOKE: &str = "desktop.revoke"; + pub const DESKTOP_WATCH: &str = "desktop.watch"; + pub const DESKTOP_UNWATCH: &str = "desktop.unwatch"; + pub const DESKTOP_STOP: &str = "desktop.stop"; + pub const DESKTOP_RESUME: &str = "desktop.resume"; pub const MESSAGES_SEND: &str = "messages.send"; pub const MESSAGES_LIST: &str = "messages.list"; pub const ATTACHMENTS_READ: &str = "attachments.read"; @@ -175,6 +179,9 @@ pub mod notification { pub const BOT_CONTEXT: &str = "bot.context"; pub const BOT_RULES: &str = "bot.rules"; pub const BOT_DESKTOP: &str = "bot.desktop"; + pub const DESKTOP_CHANGED: &str = "desktop.changed"; + /// Only to the connection watching that bot's desktop panel (spec 24.9). + pub const DESKTOP_FRAME: &str = "desktop.frame"; pub const CHAT_ITEM: &str = "chat.item"; pub const CHAT_DELTA: &str = "chat.delta"; pub const MESSAGE_CREATED: &str = "message.created"; diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs new file mode 100644 index 00000000..67bad06c --- /dev/null +++ b/crates/botloftd/src/desktop/activity.rs @@ -0,0 +1,128 @@ +//! What each bot does on the owner's desktop, for its panel (spec 24.9): +//! the window it is using, what it did last, and whether the owner stopped +//! it. A stopped bot's desktop tools refuse until the owner lets it go on. + +use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{DesktopState, DesktopWindow}; + +use crate::platform::desktop::Window; + +/// What the bot reads when the owner stopped it. +pub const STOPPED: &str = "The owner stopped you from using their desktop. Do not try again: \ + tell them in the chat what you were doing, and they let you go on when they want."; + +#[derive(Default)] +pub struct Activity { + by_bot: Mutex>, +} + +fn blank(bot: &BotId) -> DesktopState { + DesktopState { + bot_id: bot.clone(), + window: None, + action: None, + at: None, + stopped: false, + } +} + +impl Activity { + fn change(&self, bot: &BotId, how: impl FnOnce(&mut DesktopState)) -> DesktopState { + let mut all = self.by_bot.lock().unwrap_or_else(PoisonError::into_inner); + let state = all.entry(bot.clone()).or_insert_with(|| blank(bot)); + how(state); + state.clone() + } + + pub fn state(&self, bot: &BotId) -> DesktopState { + self.by_bot + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .cloned() + .unwrap_or_else(|| blank(bot)) + } + + /// The window the bot is using now, by its number. + pub fn window(&self, bot: &BotId) -> Option { + self.state(bot).window.map(|window| window.id) + } + + /// The bot read or acted in `window` at `now`; `action` says what it + /// did, when it did something. + pub fn used( + &self, + bot: &BotId, + window: &Window, + action: Option, + now: i64, + ) -> DesktopState { + self.change(bot, |state| { + state.window = Some(DesktopWindow { + id: window.id, + title: window.title.clone(), + app: window.app.name.clone(), + }); + state.action = action; + state.at = Some(now); + }) + } + + pub fn stopped(&self, bot: &BotId) -> bool { + self.state(bot).stopped + } + + /// Stops the bot, or lets it go on. + pub fn set_stopped(&self, bot: &BotId, stopped: bool) -> DesktopState { + self.change(bot, |state| state.stopped = stopped) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::platform::desktop::App; + + #[test] + fn a_bot_shows_its_window_and_last_action_and_stays_stopped_until_it_may_go_on() { + let activity = Activity::default(); + let bot = BotId::generate(); + assert_eq!(activity.state(&bot), blank(&bot)); + let window = Window { + id: 7, + title: "Visits".to_owned(), + class: "Any".to_owned(), + app: App { + path: PathBuf::from(r"C:\Apps\clinic.exe"), + name: "Clinic".to_owned(), + }, + minimized: false, + elevated: false, + }; + let used = activity.used( + &bot, + &window, + Some("Clicked button \"Save\".".to_owned()), + 9, + ); + assert_eq!(activity.window(&bot), Some(7)); + assert_eq!( + used.window.map(|window| window.app), + Some("Clinic".to_owned()) + ); + assert_eq!(used.at, Some(9)); + + assert!(!activity.stopped(&bot)); + assert!(activity.set_stopped(&bot, true).stopped); + assert!(activity.stopped(&bot)); + // Reading again does not let it go on. + activity.used(&bot, &window, None, 10); + assert!(activity.stopped(&bot)); + assert!(!activity.set_stopped(&bot, false).stopped); + } +} diff --git a/crates/botloftd/src/desktop.rs b/crates/botloftd/src/desktop/mod.rs similarity index 83% rename from crates/botloftd/src/desktop.rs rename to crates/botloftd/src/desktop/mod.rs index cd8f1b85..f1031c55 100644 --- a/crates/botloftd/src/desktop.rs +++ b/crates/botloftd/src/desktop/mod.rs @@ -1,6 +1,13 @@ //! The owner's desktop as the bots use it (spec 24): one action at a time -//! across every bot, whether the owner is there to see it, and each bot's -//! last reading, whose refs its actions name. +//! across every bot, whether the owner is there to see it, each bot's last +//! reading, whose refs its actions name, and what each bot does there, for +//! its panel, live. + +mod activity; +mod screen; + +pub use activity::{Activity, STOPPED}; +pub use screen::DesktopWatching; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -8,8 +15,10 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopView; use tokio::sync::MutexGuard; +use self::screen::Screens; use crate::platform::desktop::Control; /// How long the owner may go without touching the computer and still count @@ -34,6 +43,8 @@ pub struct Desktop { /// Held for each desktop action: the cursor is one. turn: tokio::sync::Mutex<()>, idle: Mutex, + pub activity: Arc, + screens: Arc, } /// Why the bot may not use the desktop now. @@ -68,6 +79,8 @@ impl Desktop { readings: Mutex::new(HashMap::new()), turn: tokio::sync::Mutex::new(()), idle: Mutex::new(idle), + activity: Arc::default(), + screens: Arc::default(), } } @@ -119,6 +132,19 @@ impl Desktop { .cloned() } + /// Starts watching the window the bot is using (spec 24.9). + pub fn watch(&self, bot: &BotId) -> DesktopWatching { + self.screens.watch(bot, &self.activity) + } + + /// The bot's state and the newest picture, for a panel that opens. + pub fn view(&self, bot: &BotId) -> DesktopView { + DesktopView { + state: self.activity.state(bot), + frame: self.screens.now(bot), + } + } + /// Waits for the desktop to be free; it is the caller's until dropped. pub async fn turn(&self) -> MutexGuard<'_, ()> { self.turn.lock().await diff --git a/crates/botloftd/src/desktop/screen.rs b/crates/botloftd/src/desktop/screen.rs new file mode 100644 index 00000000..5f99e161 --- /dev/null +++ b/crates/botloftd/src/desktop/screen.rs @@ -0,0 +1,100 @@ +//! The live picture of the window a bot is using, for whoever watches its +//! desktop panel (spec 24.9). While someone watches, the window is +//! pictured four times a second and a picture goes out only when it +//! changed; when the last one stops, so does the picturing. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopFrame; +use tokio::sync::watch; + +use super::activity::Activity; +use crate::platform::desktop; + +/// How often a watched window is pictured. +const EVERY: Duration = Duration::from_millis(250); + +type Frames = watch::Sender>>; + +/// Held by the connection; dropping it stops watching. +pub struct DesktopWatching { + pub bot: BotId, + /// The newest picture; `changed()` wakes on each new one. + pub frames: watch::Receiver>>, +} + +#[derive(Default)] +pub struct Screens { + /// Each watched bot's pictures, while its picturing runs. + running: Mutex>, +} + +impl Screens { + /// Starts watching the bot's window: its picturing runs while anyone + /// watches. + pub fn watch(self: &Arc, bot: &BotId, activity: &Arc) -> DesktopWatching { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(frames) = running.get(bot) { + return DesktopWatching { + bot: bot.clone(), + frames: frames.subscribe(), + }; + } + let (frames, watching) = watch::channel(None); + running.insert(bot.clone(), frames.clone()); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let (screens, activity, bot) = (Arc::clone(self), Arc::clone(activity), bot.clone()); + runtime.spawn(async move { screens.picture(&bot, &activity, &frames).await }); + } + DesktopWatching { + bot: bot.clone(), + frames: watching, + } + } + + /// The newest picture of the bot's window, while it is watched. + pub fn now(&self, bot: &BotId) -> Option { + self.running + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .and_then(|frames| frames.borrow().as_deref().cloned()) + } + + /// Pictures the bot's window until nobody watches. + async fn picture(&self, bot: &BotId, activity: &Activity, frames: &Frames) { + let mut last: Option = None; + loop { + { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if frames.receiver_count() == 0 { + running.remove(bot); + return; + } + } + if let Some(id) = activity.window(bot) { + let picture = tokio::task::spawn_blocking(move || desktop::picture(id)).await; + if let Ok(Ok(picture)) = picture + && !picture.jpeg.is_empty() + { + let data = BASE64.encode(&picture.jpeg); + if last.as_deref() != Some(data.as_str()) { + last = Some(data.clone()); + frames.send_replace(Some(Arc::new(DesktopFrame { + bot_id: bot.clone(), + data, + width: picture.width, + height: picture.height, + }))); + } + } + } + tokio::time::sleep(EVERY).await; + } + } +} diff --git a/crates/botloftd/src/main.rs b/crates/botloftd/src/main.rs index e60871c2..c14c0660 100644 --- a/crates/botloftd/src/main.rs +++ b/crates/botloftd/src/main.rs @@ -22,7 +22,7 @@ use botloftd::trash::RecycleBin; use botloftd::{approvals, autostart, keep_awake, logging, routines, secrets, server}; use clap::{Parser, Subcommand}; use tokio::net::TcpListener; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[derive(Parser)] #[command(name = "botloftd", version, about = "Botloft daemon")] @@ -137,6 +137,12 @@ fn run(paths: Paths, config: Config, config_path: PathBuf, scheduled: bool) -> a }); // No bot process survived the last run, so nobody waits for these. approvals::expire_all(&daemon); + // Ctrl+Alt+End stops every bot on the desktop, app open or not (spec 24.9). + let stopping = Arc::clone(&daemon); + match platform::desktop::on_stop_key(move || botloftd::service::desktop::stop_all(&stopping)) { + Ok(()) | Err(platform::desktop::DesktopError::Unavailable) => {} + Err(err) => warn!("the shortcut that stops bots on the desktop is not available: {err}"), + } let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 3de60059..73096570 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -126,6 +126,20 @@ pub fn windows() -> Result, DesktopError> { } } +/// Calls `stop` each time the owner presses Ctrl+Alt+End, from now on +/// (spec 24.9). Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + #[cfg(windows)] + { + windows::on_stop_key(stop) + } + #[cfg(not(windows))] + { + let _ = stop; + Err(DesktopError::Unavailable) + } +} + /// How long ago the owner last used the mouse or keyboard (spec 24.8); /// `None` where that cannot be known. pub fn owner_idle() -> Option { diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 64207d3c..c7c82c76 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -8,6 +8,7 @@ mod act_tests; mod capture; mod input; mod process; +mod stop_key; #[cfg(test)] mod test_window; #[cfg(test)] @@ -31,6 +32,7 @@ use windows::core::{BOOL, w}; pub use act::act; pub use capture::capture; pub use input::owner_idle; +pub use stop_key::on_stop_key; pub use uia::read; use super::{App, DesktopError, Window}; diff --git a/crates/botloftd/src/platform/desktop/windows/stop_key.rs b/crates/botloftd/src/platform/desktop/windows/stop_key.rs new file mode 100644 index 00000000..b70cde46 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/stop_key.rs @@ -0,0 +1,52 @@ +//! The shortcut that stops every bot on the desktop, Ctrl+Alt+End (spec +//! 24.9): registered for the whole session on a thread of its own, so it +//! works with the app closed. + +use std::sync::mpsc; + +use windows::Win32::UI::Input::KeyboardAndMouse::{ + MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, RegisterHotKey, VK_END, +}; +use windows::Win32::UI::WindowsAndMessaging::{GetMessageW, MSG, WM_HOTKEY}; + +use super::super::DesktopError; + +/// The shortcut's id for this thread. +const STOP: i32 = 1; + +/// Calls `stop` each time the owner presses the shortcut, from now on. +/// Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + let (told, registered) = mpsc::channel(); + std::thread::Builder::new() + .name("desktop-stop-key".to_owned()) + .spawn(move || { + // SAFETY: a shortcut for this thread's own message queue. + let held = unsafe { + RegisterHotKey( + None, + STOP, + MOD_CONTROL | MOD_ALT | MOD_NOREPEAT, + u32::from(VK_END.0), + ) + }; + let ok = held.is_ok(); + let _ = told.send(held.map_err(|err| DesktopError::System(err.message()))); + if !ok { + return; + } + let mut message = MSG::default(); + // SAFETY: this thread's own queue, read until the process ends. + while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() { + if message.message == WM_HOTKEY && message.wParam.0 == STOP as usize { + stop(); + } + } + }) + .map_err(|err| DesktopError::System(err.to_string()))?; + registered.recv().unwrap_or_else(|_| { + Err(DesktopError::System( + "the shortcut's thread ended".to_owned(), + )) + }) +} diff --git a/crates/botloftd/src/rpc/desktop_watch.rs b/crates/botloftd/src/rpc/desktop_watch.rs new file mode 100644 index 00000000..9390c5f6 --- /dev/null +++ b/crates/botloftd/src/rpc/desktop_watch.rs @@ -0,0 +1,50 @@ +//! A connection watching one bot's desktop panel (spec 24.9): +//! `desktop.watch` and `desktop.unwatch`, and the pictures that follow. + +use botloft_core::protocol::{DesktopBotParams, error_code, method, notification}; +use serde_json::Value; + +use super::dispatch::parse; +use super::jsonrpc::{self, RpcError}; +use crate::desktop::DesktopWatching; +use crate::service::bots; +use crate::state::Daemon; + +#[derive(Default)] +pub(super) struct DesktopWatch { + current: Option, +} + +impl DesktopWatch { + pub(super) fn request( + &mut self, + daemon: &Daemon, + name: &str, + params: Option, + ) -> Result { + // Stop the one before, even if the new one is refused. + self.current = None; + if name == method::DESKTOP_UNWATCH { + return Ok(Value::Null); + } + let params: DesktopBotParams = parse(params)?; + bots::find(&daemon.store(), ¶ms.bot_id).map_err(RpcError::from)?; + self.current = Some(daemon.desktop.watch(¶ms.bot_id)); + serde_json::to_value(daemon.desktop.view(¶ms.bot_id)) + .map_err(|err| RpcError::new(error_code::INTERNAL_ERROR, err.to_string())) + } + + /// The watched window's next picture, as a notification. Never resolves + /// while nothing is watched. + pub(super) async fn next_frame(&mut self) -> Option { + let Some(watching) = self.current.as_mut() else { + return std::future::pending().await; + }; + if watching.frames.changed().await.is_err() { + self.current = None; + return None; + } + let frame = watching.frames.borrow_and_update().clone()?; + Some(jsonrpc::notification(notification::DESKTOP_FRAME, &*frame)) + } +} diff --git a/crates/botloftd/src/rpc/dispatch.rs b/crates/botloftd/src/rpc/dispatch.rs index f1c64d06..035111b5 100644 --- a/crates/botloftd/src/rpc/dispatch.rs +++ b/crates/botloftd/src/rpc/dispatch.rs @@ -51,6 +51,8 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option) -> Result reply(rules::delete(daemon, parse(params)?)), method::DESKTOP_GRANTS => reply(desktop::grants(daemon, parse(params)?)), method::DESKTOP_REVOKE => reply(desktop::revoke(daemon, parse(params)?)), + method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)), + method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)), method::MESSAGES_SEND => reply(messages::send(daemon, parse(params)?)), method::MESSAGES_LIST => reply(messages::list(daemon, parse(params)?)), method::ATTACHMENTS_READ => reply(attachments::read(daemon, parse(params)?)), diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index d92ba5cb..3085aa46 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -1,24 +1,26 @@ //! The app's connection: one WebSocket per client, JSON-RPC 2.0 over text //! frames (spec 11). The first request must be `session.hello`. +mod desktop_watch; mod dispatch; pub mod jsonrpc; +mod notify; mod watching; use std::sync::Arc; use std::time::Duration; use axum::extract::ws::{Message, WebSocket}; -use botloft_core::protocol::{ - HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method, notification, -}; +use botloft_core::protocol::{HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method}; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use serde_json::Value; use tokio::sync::{broadcast, mpsc, watch}; use tracing::{debug, warn}; +use self::desktop_watch::DesktopWatch; use self::jsonrpc::RpcError; +use self::notify::to_notification; use self::watching::Watch; use crate::state::{Daemon, Event}; @@ -140,10 +142,11 @@ async fn run( daemon: &Arc, ) { let mut watch = Watch::default(); + let mut desktop = DesktopWatch::default(); loop { let frame = tokio::select! { message = stream.next() => match message { - Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, frames) { + Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, &mut desktop, frames) { Routed::Reply(frame) => frame, Routed::InOrder(request) => answer(daemon, request).await, Routed::Aside(request) => { @@ -167,6 +170,10 @@ async fn run( frames.send_replace(frame); None } + frame = desktop.next_frame() => { + frames.send_replace(frame); + None + } }; if let Some(frame) = frame && outbox.send(frame).await.is_err() @@ -201,6 +208,7 @@ fn route( daemon: &Arc, text: &str, watch: &mut Watch, + desktop: &mut DesktopWatch, frames: &watch::Sender>, ) -> Routed { let request = match jsonrpc::parse(text) { @@ -228,6 +236,11 @@ fn route( | method::BROWSER_SWITCH_TAB | method::BROWSER_OPEN | method::BROWSER_TEACH => watch.request(daemon, &request.method, request.params), + // So is watching a bot's desktop panel (spec 24.9). + method::DESKTOP_WATCH | method::DESKTOP_UNWATCH => { + frames.send_replace(None); + desktop.request(daemon, &request.method, request.params) + } // A window of its own outlives the request (spec 21.11). method::BROWSER_WINDOW => watch.window(daemon, request.params), name if ASIDE.contains(&name) => return Routed::Aside(request), @@ -271,30 +284,3 @@ fn reply(id: &Value, result: Result) -> String { Err(err) => jsonrpc::failure(id, &err), } } - -fn to_notification(event: &Event) -> String { - use jsonrpc::notification as note; - match event { - Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), - Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), - Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), - Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), - Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), - Event::BotState(state) => note(notification::BOT_STATE, state), - Event::BotContext(context) => note(notification::BOT_CONTEXT, context), - Event::BotRules(rules) => note(notification::BOT_RULES, rules), - Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), - Event::ChatItem(item) => note(notification::CHAT_ITEM, item), - Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), - Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), - Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), - Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), - Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), - Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), - Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), - Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), - Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), - Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), - Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), - } -} diff --git a/crates/botloftd/src/rpc/notify.rs b/crates/botloftd/src/rpc/notify.rs new file mode 100644 index 00000000..1c2d65f5 --- /dev/null +++ b/crates/botloftd/src/rpc/notify.rs @@ -0,0 +1,34 @@ +//! What changed, as the notification the apps get (spec 11). + +use botloft_core::protocol::notification; + +use super::jsonrpc; +use crate::state::Event; + +pub(super) fn to_notification(event: &Event) -> String { + use jsonrpc::notification as note; + match event { + Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), + Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), + Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), + Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), + Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), + Event::BotState(state) => note(notification::BOT_STATE, state), + Event::BotContext(context) => note(notification::BOT_CONTEXT, context), + Event::BotRules(rules) => note(notification::BOT_RULES, rules), + Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), + Event::DesktopChanged(state) => note(notification::DESKTOP_CHANGED, state), + Event::ChatItem(item) => note(notification::CHAT_ITEM, item), + Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), + Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), + Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), + Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), + Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), + Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), + Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), + Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), + Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), + Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), + Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), + } +} diff --git a/crates/botloftd/src/service/desktop.rs b/crates/botloftd/src/service/desktop.rs index 2bdc2b7c..4d44e656 100644 --- a/crates/botloftd/src/service/desktop.rs +++ b/crates/botloftd/src/service/desktop.rs @@ -1,9 +1,15 @@ //! What each bot may see and do on the owner's desktop (spec 24.2, //! 24.10): `desktop.grants` and `desktop.revoke`. Grants come from the -//! owner answering a bot's request (`tools/desktop.rs`). +//! owner answering a bot's request (`tools/desktop.rs`). And stopping a +//! bot there, or every bot at once (spec 24.9): `desktop.stop` and +//! `desktop.resume`. use botloft_core::ids::BotId; -use botloft_core::protocol::{BotDesktop, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams}; +use botloft_core::protocol::{ + BotDesktop, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams, + DesktopState, +}; +use tracing::{info, warn}; use super::{ApiError, ApiResult}; use crate::state::{Daemon, Event}; @@ -34,3 +40,39 @@ pub(crate) fn changed(daemon: &Daemon, bot_id: &BotId) -> ApiResult daemon.emit(Event::BotDesktop(desktop.clone())); Ok(desktop) } + +/// Stops the bot on the desktop, or lets it go on. +fn set_stopped(daemon: &Daemon, bot: &BotId, stopped: bool) -> DesktopState { + let state = daemon.desktop.activity.set_stopped(bot, stopped); + daemon.emit(Event::DesktopChanged(state.clone())); + state +} + +pub fn stop(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, true)) +} + +pub fn resume(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, false)) +} + +/// Stops every bot on the desktop: the owner pressed the stop shortcut +/// (spec 24.9). +pub fn stop_all(daemon: &Daemon) { + let bots = match daemon.store().bots(None, false) { + Ok(bots) => bots, + Err(err) => { + warn!("could not list the bots to stop them on the desktop: {err}"); + return; + } + }; + for bot in &bots { + set_stopped(daemon, &bot.id, true); + } + info!( + bots = bots.len(), + "the owner stopped every bot on the desktop" + ); +} diff --git a/crates/botloftd/src/state.rs b/crates/botloftd/src/state.rs index b9768d08..f4578162 100644 --- a/crates/botloftd/src/state.rs +++ b/crates/botloftd/src/state.rs @@ -8,7 +8,8 @@ use std::time::Instant; use botloft_core::protocol::{ AccountUsage, Bot, BotContextChanged, BotDeleted, BotDesktop, BotRules, BotStateChanged, BrowserAction, BrowserState, ChatDelta, ChatItemChanged, Crew, CrewDeleted, Delivery, - FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, ScreenDraft, Task, + DesktopState, FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, + ScreenDraft, Task, }; use botloft_store::Store; use tokio::runtime::{Handle, RuntimeFlavor}; @@ -43,6 +44,7 @@ pub enum Event { BotContext(BotContextChanged), BotRules(BotRules), BotDesktop(BotDesktop), + DesktopChanged(DesktopState), ChatItem(ChatItemChanged), ChatDelta(ChatDelta), MessageCreated(Message), diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index 3867e14e..08464846 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -15,8 +15,10 @@ use super::calls::explain; use super::desktop_act::{Ask, act}; use super::desktop_grant::{blocking, granted, heading}; use super::desktop_list::list; +use crate::desktop::STOPPED; +use crate::platform::desktop::Window; use crate::platform::desktop::{self as platform, Action, Scroll, render}; -use crate::state::Daemon; +use crate::state::{Daemon, Event}; pub(super) const PREFIX: &str = "desktop_"; @@ -136,6 +138,9 @@ pub(super) async fn call( } async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Result { + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } daemon .desktop .owner_here() @@ -161,10 +166,14 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let controls = blocking(move || platform::read(id)).await?; let (text, next) = render(&controls, from); daemon.desktop.keep(bot, id, controls); + used(daemon, bot, &window, None); let mut reading = format!("{}\n{text}", heading(&window)); if let Some(next) = next { reading.push_str(&format!( @@ -186,8 +195,12 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let picture = blocking(move || platform::picture(id)).await?; + used(daemon, bot, &window, None); if picture.jpeg.is_empty() { return Err("The picture of that window came out empty.".to_owned()); } @@ -201,3 +214,10 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul .map(Reply::Text), } } + +/// The bot read or acted in `window`: its panel shows it (spec 24.9). +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { + let now = daemon.clock.now_ms(); + let state = daemon.desktop.activity.used(bot, window, action, now); + daemon.emit(Event::DesktopChanged(state)); +} diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index 29db9662..cf1cec8c 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -8,7 +8,9 @@ use std::time::Duration; use botloft_core::ids::BotId; use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use super::desktop::used; use super::desktop_grant::{blocking, describe, granted, heading}; +use crate::desktop::STOPPED; use crate::platform::desktop::{ self as platform, Acted, Action, Control, DesktopError, Window, by_reference, render, }; @@ -73,6 +75,10 @@ pub(super) async fn act( ) .await?; let _turn = daemon.desktop.turn().await; + // The owner may have stopped it while it waited. + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let (id, target, action) = (window.id, control.runtime_id.clone(), ask.action.clone()); let acted = blocking(move || platform::act(id, &target, &action)).await?; tokio::time::sleep(SETTLE).await; @@ -86,6 +92,7 @@ pub(super) async fn act( let (text, next) = render(&controls, 0); daemon.desktop.keep(bot, id, controls); let mut answer = done(&ask.action, &control); + used(daemon, bot, &window, Some(answer.clone())); if acted == Acted::Waiting { answer.push_str( " The app did not answer in time: it may be showing a dialog. Call desktop_windows \ diff --git a/crates/botloftd/tests/common/desktop.rs b/crates/botloftd/tests/common/desktop.rs new file mode 100644 index 00000000..ca9415ec --- /dev/null +++ b/crates/botloftd/tests/common/desktop.rs @@ -0,0 +1,169 @@ +//! The desktop tests' own window (spec 24): a Save button that renames it, +//! a field and a password field, on a thread with its message loop; and a +//! daemon with a crew and a bot ready to use it. Windows only. + +use std::sync::mpsc; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::{Value, json}; +use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::WindowsAndMessaging::{ + BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, + GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, + TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, + WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, +}; +use windows::core::{PCWSTR, w}; + +use super::bots::ready_bot; +use super::mcp::Mcp; +use super::{Client, TestDaemon}; + +pub const TITLE: &str = "Botloft desktop: the clinic"; + +unsafe extern "system" fn procedure( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if message == WM_DESTROY { + // SAFETY: ends this thread's message loop. + unsafe { PostQuitMessage(0) }; + return LRESULT(0); + } + // The Save button (id 1) clicked: the window says so in its title. + if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { + // SAFETY: renames our own window, on its thread. + let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; + return LRESULT(0); + } + // SAFETY: everything else as Windows does by default. + unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } +} + +/// A window of the test's own, on a thread with its message loop. +pub struct TestWindow { + pub id: u64, + thread: Option>, +} + +impl TestWindow { + pub fn open() -> Self { + let (sent, opened) = mpsc::channel(); + let thread = std::thread::spawn(move || { + // SAFETY: a window and its controls on this thread, with its loop. + unsafe { + let class = WNDCLASSW { + lpfnWndProc: Some(procedure), + hInstance: GetModuleHandleW(None).expect("module").into(), + lpszClassName: w!("BotloftDesktopToolsTest"), + ..Default::default() + }; + RegisterClassW(&class); + let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + w!("BotloftDesktopToolsTest"), + PCWSTR(title.as_ptr()), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + CW_USEDEFAULT, + CW_USEDEFAULT, + 420, + 320, + None, + None, + None, + None, + ) + .expect("window"); + let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { + CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + text, + WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), + 10, + top, + 160, + 24, + Some(hwnd), + Some(HMENU(id as *mut std::ffi::c_void)), + None, + None, + ) + .expect("control"); + }; + child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); + child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); + child( + w!("EDIT"), + w!("hunter2"), + WS_BORDER.0 | ES_PASSWORD as u32, + 70, + 3, + ); + sent.send(hwnd.0 as usize as u64).expect("send"); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + }); + let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); + std::thread::sleep(Duration::from_millis(300)); + Self { + id, + thread: Some(thread), + } + } +} + +impl Drop for TestWindow { + fn drop(&mut self) { + let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); + // SAFETY: asks the window's own thread to close it. + let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +pub struct Setup { + pub t: TestDaemon, + pub app: Client, + pub bot: Value, + pub mcp: Mcp, +} + +pub async fn setup() -> Setup { + let t = TestDaemon::start_supervised().await; + let mut app = t.session().await; + let crew = app + .call("crews.create", json!({ "name": "Clinic" })) + .await + .expect("crew"); + let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; + Setup { t, app, bot, mcp } +} + +pub fn reading(result: &Result) -> &str { + match result { + Ok(text) | Err(text) => text, + } +} + +/// The ref of the line that has `what` in a reading. +pub fn ref_in(reading: &str, what: &str) -> String { + let line = reading + .lines() + .find(|line| line.contains(what)) + .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); + let start = line.find('[').expect("a ref") + 1; + let end = line.find(']').expect("a ref"); + line[start..end].to_owned() +} diff --git a/crates/botloftd/tests/common/mod.rs b/crates/botloftd/tests/common/mod.rs index 7a9dfe84..bf81669f 100644 --- a/crates/botloftd/tests/common/mod.rs +++ b/crates/botloftd/tests/common/mod.rs @@ -6,6 +6,8 @@ pub mod bots; pub mod browsing; pub mod context; +#[cfg(windows)] +pub mod desktop; pub mod mcp; pub mod routines; pub mod site; diff --git a/crates/botloftd/tests/desktop.rs b/crates/botloftd/tests/desktop.rs index 466b3710..0e1a405f 100644 --- a/crates/botloftd/tests/desktop.rs +++ b/crates/botloftd/tests/desktop.rs @@ -6,161 +6,11 @@ mod common; -use std::sync::mpsc; -use std::thread::JoinHandle; use std::time::Duration; -use common::Client; -use common::TestDaemon; -use common::bots::ready_bot; use common::browsing::{call, pending_approval}; -use common::mcp::Mcp; -use serde_json::{Value, json}; -use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; -use windows::Win32::System::LibraryLoader::GetModuleHandleW; -use windows::Win32::UI::WindowsAndMessaging::{ - BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, - GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, - TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, - WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, -}; -use windows::core::{PCWSTR, w}; - -const TITLE: &str = "Botloft desktop: the clinic"; - -unsafe extern "system" fn procedure( - hwnd: HWND, - message: u32, - wparam: WPARAM, - lparam: LPARAM, -) -> LRESULT { - if message == WM_DESTROY { - // SAFETY: ends this thread's message loop. - unsafe { PostQuitMessage(0) }; - return LRESULT(0); - } - // The Save button (id 1) clicked: the window says so in its title. - if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { - // SAFETY: renames our own window, on its thread. - let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; - return LRESULT(0); - } - // SAFETY: everything else as Windows does by default. - unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } -} - -/// A window of the test's own, on a thread with its message loop. -struct TestWindow { - id: u64, - thread: Option>, -} - -impl TestWindow { - fn open() -> Self { - let (sent, opened) = mpsc::channel(); - let thread = std::thread::spawn(move || { - // SAFETY: a window and its controls on this thread, with its loop. - unsafe { - let class = WNDCLASSW { - lpfnWndProc: Some(procedure), - hInstance: GetModuleHandleW(None).expect("module").into(), - lpszClassName: w!("BotloftDesktopToolsTest"), - ..Default::default() - }; - RegisterClassW(&class); - let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); - let hwnd = CreateWindowExW( - WINDOW_EX_STYLE::default(), - w!("BotloftDesktopToolsTest"), - PCWSTR(title.as_ptr()), - WS_OVERLAPPEDWINDOW | WS_VISIBLE, - CW_USEDEFAULT, - CW_USEDEFAULT, - 420, - 320, - None, - None, - None, - None, - ) - .expect("window"); - let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { - CreateWindowExW( - WINDOW_EX_STYLE::default(), - class, - text, - WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), - 10, - top, - 160, - 24, - Some(hwnd), - Some(HMENU(id as *mut std::ffi::c_void)), - None, - None, - ) - .expect("control"); - }; - child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); - child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); - child( - w!("EDIT"), - w!("hunter2"), - WS_BORDER.0 | ES_PASSWORD as u32, - 70, - 3, - ); - sent.send(hwnd.0 as usize as u64).expect("send"); - let mut message = MSG::default(); - while GetMessageW(&mut message, None, 0, 0).as_bool() { - let _ = TranslateMessage(&message); - DispatchMessageW(&message); - } - } - }); - let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); - std::thread::sleep(Duration::from_millis(300)); - Self { - id, - thread: Some(thread), - } - } -} - -impl Drop for TestWindow { - fn drop(&mut self) { - let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); - // SAFETY: asks the window's own thread to close it. - let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } - } -} - -struct Setup { - t: TestDaemon, - app: Client, - bot: Value, - mcp: Mcp, -} - -async fn setup() -> Setup { - let t = TestDaemon::start_supervised().await; - let mut app = t.session().await; - let crew = app - .call("crews.create", json!({ "name": "Clinic" })) - .await - .expect("crew"); - let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; - Setup { t, app, bot, mcp } -} - -fn reading(result: &Result) -> &str { - match result { - Ok(text) | Err(text) => text, - } -} +use common::desktop::{TITLE, TestWindow, reading, ref_in, setup}; +use serde_json::json; #[tokio::test(flavor = "multi_thread")] async fn a_bot_sees_an_app_once_the_owner_lets_it_and_never_a_password() { @@ -251,17 +101,6 @@ async fn nothing_is_seen_while_the_owner_is_away() { ); } -/// The ref of the line that has `what` in a reading. -fn ref_in(reading: &str, what: &str) -> String { - let line = reading - .lines() - .find(|line| line.contains(what)) - .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); - let start = line.find('[').expect("a ref") + 1; - let end = line.find(']').expect("a ref"); - line[start..end].to_owned() -} - #[tokio::test(flavor = "multi_thread")] async fn a_bot_uses_an_app_once_the_owner_lets_it_and_never_types_a_password() { let window = TestWindow::open(); diff --git a/crates/botloftd/tests/desktop_panel.rs b/crates/botloftd/tests/desktop_panel.rs new file mode 100644 index 00000000..b4f2cc7b --- /dev/null +++ b/crates/botloftd/tests/desktop_panel.rs @@ -0,0 +1,83 @@ +//! A bot's desktop panel with real windows (spec 24.9): the window it is +//! using and what it did, its pictures while the panel is open, and the +//! owner stopping it, one bot or all of them. Windows only. +#![cfg(windows)] + +mod common; + +use common::browsing::{call, pending_approval}; +use common::desktop::{TITLE, TestWindow, reading, setup}; +use serde_json::json; + +#[tokio::test(flavor = "multi_thread")] +async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { + let window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + // Nothing used yet: no window, no picture. + let view = s + .app + .call("desktop.watch", id.clone()) + .await + .expect("watch"); + assert_eq!(view["state"]["window"], serde_json::Value::Null); + assert_eq!(view["state"]["stopped"], false); + assert_eq!(view["frame"], serde_json::Value::Null); + + let look = json!({ "window": window.id, "why": "To read the visit" }); + let looking = call(&s.mcp, "desktop_look", look); + let asked = pending_approval(&mut s.app).await; + let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); + s.app.call("approvals.answer", allow).await.expect("answer"); + looking.await.expect("task").expect("reading"); + + // The panel hears which window, and its picture follows. + let changed = s.app.notification("desktop.changed").await; + assert_eq!(changed["window"]["title"], TITLE); + assert_eq!(changed["action"], serde_json::Value::Null); + let frame = s.app.notification("desktop.frame").await; + assert_eq!(frame["botId"], s.bot["id"]); + assert!(frame["data"].as_str().is_some_and(|data| data.len() > 100)); + + s.app + .call("desktop.unwatch", serde_json::Value::Null) + .await + .expect("unwatch"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_stopped_bot_cannot_use_the_desktop_until_the_owner_lets_it_go_on() { + let _window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + let stopped = s.app.call("desktop.stop", id.clone()).await.expect("stop"); + assert_eq!(stopped["stopped"], true); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); + + let going = s + .app + .call("desktop.resume", id.clone()) + .await + .expect("resume"); + assert_eq!(going["stopped"], false); + s.mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows again"); + + // The shortcut stops every bot at once. + botloftd::service::desktop::stop_all(&s.t.daemon); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); +} diff --git a/docs/spec.md b/docs/spec.md index 7dcb5f18..2c64c4a2 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1653,7 +1653,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 e D2 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1 e D2 implementados; D3 em andamento** (24.12): o daemon já guarda o que cada bot faz no desktop, manda a foto ao vivo, para e deixa continuar, e tem o atalho; falta o painel no app. É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1755,8 +1755,11 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela em que o bot está agindo (`PrintWindow` a cada mudança, no máximo 5 por segundo, só enquanto o painel está aberto), o nome do app, e a linha do que ele acabou de fazer. Um botão **Parar** encerra o que o bot faz no desktop na hora. -- **Parar tudo**: um atalho global, `Ctrl+Alt+End`, que o daemon registra (`RegisterHotKey`), para qualquer ação de desktop de qualquer bot, mesmo com o app fechado. +- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação como a tool disse (`Clicked button "Save".`; nada depois de só ler), quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. +- **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. +- **Parar tudo:** o atalho global `Ctrl+Alt+End`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele. - O cartão de permissão no chat (24.2) diz " quer ver " com o ícone de tela, o porquê do bot (" diz: ..."), o caminho do executável e o que ver deixa fazer (ler as janelas do app e fotografá-las com o dono no computador, nunca senhas, e onde tirar). Respondido, vira uma linha: " pode ver ", " não pode ver " com a nota, ou sem resposta. As tools `desktop_*` aparecem no chat com o ícone de tela e o porquê como resumo. ### 24.10 Permissões no app @@ -1784,7 +1787,7 @@ No servidor `botloft` (11), como as do navegador. |---|---|---| | **D1** Ver | `desktop_grants`, permissões pelo chat (app, Ver), o que nunca é liberado, `desktop_windows`, `desktop_look`, `desktop_screenshot` de uma janela, a seção Desktop nas configurações do bot | unidade; Windows de verdade com uma janela de teste própria (Win32 com botões e campos, inclusive de senha) | | **D2** Mexer | nível Mexer, `desktop_click`, `desktop_type`, `desktop_select`, `desktop_scroll` por acessibilidade, uma ação por vez | a janela de teste e o Bloco de Notas | -| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | app com FakeBotloft; o atalho no Windows de verdade | +| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | o daemon com uma janela de verdade (foto ao vivo, parar, deixar continuar, parar todos); app com FakeBotloft; o atalho apertado de verdade, à mão | | **D4** Mouse e teclado | a opção, `SendInput`, `desktop_press`, `desktop_click_at`, o dono assumindo pelo gancho, o aviso na tela | a janela de teste; o dono simulado por entrada não injetada | | **D5** Sem você | a opção, a tela de riscos, o aviso na volta, a regra dos 5 minutos, tela bloqueada | unidade; teste manual com a tela bloqueada | | **D6** Desktop inteiro | o alcance, a foto da tela com as janelas bloqueadas cobertas | a janela de teste e uma bloqueada | From 085d7ac790dc72082517cd03c36b5d740ebc2442 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 19:45:23 -0300 Subject: [PATCH 2/2] fix(desktop): the panel words the bot's action itself, and no app holds a reading for minutes DesktopState.action is now structured (DesktopAction: click, type, select or scroll, the control's name and the option), so the app words it in the owner's language instead of showing the tool's English. Every UI Automation session, reading too, gets 5 seconds per call and to connect, and a walk stops at 15 seconds with what it read: right after a classic combo box took a choice through its default action, the window could answer slowly, and the next reading, without a limit, hung for minutes. The choice and scroll test waits up to 2 seconds for the window to show them, as the tool does; 12 runs in a row pass. --- app/src/lib/fakeDesktop.ts | 3 +- app/src/lib/protocol.gen.ts | 21 ++++++++++-- crates/botloft-core/src/protocol/desktop.rs | 28 ++++++++++++++-- .../src/protocol/export_desktop.rs | 2 ++ crates/botloftd/src/desktop/activity.rs | 16 +++++----- .../src/platform/desktop/windows/act.rs | 23 +++++-------- .../src/platform/desktop/windows/act_tests.rs | 32 +++++++++++++++---- .../src/platform/desktop/windows/uia.rs | 27 +++++++++++++--- crates/botloftd/src/tools/desktop.rs | 4 +-- crates/botloftd/src/tools/desktop_act.rs | 19 +++++++++-- docs/spec.md | 4 +-- 11 files changed, 131 insertions(+), 48 deletions(-) diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index fabd6cbc..6f0e1de1 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -8,6 +8,7 @@ import type { FakeBotloft, Handlers } from "./fake"; import { notFound } from "./fakeRules"; import type { BotId, + DesktopAction, DesktopFrame, DesktopGrant, DesktopLevel, @@ -74,7 +75,7 @@ export class FakeDesktop { } /** The bot read or acted in `window`; `action` says what it did. */ - use(botId: BotId, window: DesktopWindow, action: string | null = null): DesktopState { + use(botId: BotId, window: DesktopWindow, action: DesktopAction | null = null): DesktopState { return this.set(botId, { window, action, at: this.fake.now }); } diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index a5c29847..a54e9090 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -878,6 +878,22 @@ export type DesktopWindow = { id: number, title: string, */ app: string, }; +/** + * What a bot did in a window (spec 24.5), for its panel, which words it + * in the owner's language. + */ +export type DesktopActionKind = "click" | "type" | "select" | "scroll"; + +export type DesktopAction = { kind: DesktopActionKind, +/** + * The control's name as the window reads it; empty when it has none. + */ +target: string, +/** + * The option chosen, for `select`. + */ +option: string | null, }; + /** * What a bot does on the owner's desktop, for its panel (spec 24.9): the * params of `desktop.changed`. @@ -888,10 +904,9 @@ export type DesktopState = { botId: BotId, */ window: DesktopWindow | null, /** - * What it did last, in English, as its tool said ("Clicked button - * \"Save\"."); `null` after only reading. + * What it did last; `null` after only reading. */ -action: string | null, +action: DesktopAction | null, /** * When it last read or acted, Unix ms. */ diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 29cc7b97..32f88849 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -119,6 +119,29 @@ pub struct DesktopWindow { pub app: String, } +/// What a bot did in a window (spec 24.5), for its panel, which words it +/// in the owner's language. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub enum DesktopActionKind { + Click, + Type, + Select, + Scroll, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopAction { + pub kind: DesktopActionKind, + /// The control's name as the window reads it; empty when it has none. + pub target: String, + /// The option chosen, for `select`. + pub option: Option, +} + /// What a bot does on the owner's desktop, for its panel (spec 24.9): the /// params of `desktop.changed`. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -128,9 +151,8 @@ pub struct DesktopState { pub bot_id: BotId, /// The window it read or acted in last. pub window: Option, - /// What it did last, in English, as its tool said ("Clicked button - /// \"Save\"."); `null` after only reading. - pub action: Option, + /// What it did last; `null` after only reading. + pub action: Option, /// When it last read or acted, Unix ms. pub at: Option, /// The owner stopped it (spec 24.9): its desktop tools refuse until diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index 131f090b..0230b6ca 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -13,6 +13,8 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); out.decl::(); out.decl::(); out.decl::(); diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs index 67bad06c..3c5d111a 100644 --- a/crates/botloftd/src/desktop/activity.rs +++ b/crates/botloftd/src/desktop/activity.rs @@ -6,7 +6,7 @@ use std::collections::HashMap; use std::sync::{Mutex, PoisonError}; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopState, DesktopWindow}; +use botloft_core::protocol::{DesktopAction, DesktopState, DesktopWindow}; use crate::platform::desktop::Window; @@ -57,7 +57,7 @@ impl Activity { &self, bot: &BotId, window: &Window, - action: Option, + action: Option, now: i64, ) -> DesktopState { self.change(bot, |state| { @@ -104,12 +104,12 @@ mod tests { minimized: false, elevated: false, }; - let used = activity.used( - &bot, - &window, - Some("Clicked button \"Save\".".to_owned()), - 9, - ); + let click = DesktopAction { + kind: botloft_core::protocol::DesktopActionKind::Click, + target: "Save".to_owned(), + option: None, + }; + let used = activity.used(&bot, &window, Some(click), 9); assert_eq!(activity.window(&bot), Some(7)); assert_eq!( used.window.map(|window| window.app), diff --git a/crates/botloftd/src/platform/desktop/windows/act.rs b/crates/botloftd/src/platform/desktop/windows/act.rs index bbff3cb8..1928edb5 100644 --- a/crates/botloftd/src/platform/desktop/windows/act.rs +++ b/crates/botloftd/src/platform/desktop/windows/act.rs @@ -4,23 +4,20 @@ use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - ExpandCollapseState_Collapsed, IUIAutomation2, IUIAutomationElement, - IUIAutomationExpandCollapsePattern, IUIAutomationInvokePattern, - IUIAutomationLegacyIAccessiblePattern, IUIAutomationScrollItemPattern, - IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, IUIAutomationTogglePattern, - IUIAutomationValuePattern, ScrollAmount_LargeDecrement, ScrollAmount_LargeIncrement, - ScrollAmount_NoAmount, TreeScope_Descendants, UIA_ExpandCollapsePatternId, UIA_InvokePatternId, - UIA_LegacyIAccessiblePatternId, UIA_NamePropertyId, UIA_ScrollItemPatternId, - UIA_ScrollPatternId, UIA_SelectionItemPatternId, UIA_TogglePatternId, UIA_ValuePatternId, + ExpandCollapseState_Collapsed, IUIAutomationElement, IUIAutomationExpandCollapsePattern, + IUIAutomationInvokePattern, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationScrollItemPattern, IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, + IUIAutomationTogglePattern, IUIAutomationValuePattern, ScrollAmount_LargeDecrement, + ScrollAmount_LargeIncrement, ScrollAmount_NoAmount, TreeScope_Descendants, + UIA_ExpandCollapsePatternId, UIA_InvokePatternId, UIA_LegacyIAccessiblePatternId, + UIA_NamePropertyId, UIA_ScrollItemPatternId, UIA_ScrollPatternId, UIA_SelectionItemPatternId, + UIA_TogglePatternId, UIA_ValuePatternId, }; use windows::core::{BSTR, Interface}; use super::super::{Acted, Action, DesktopError, Scroll}; use super::uia::{Session, system}; -/// How long a call into the app may take: one that opens a dialog may not -/// come back until the dialog closes. -const TRANSACTION_MS: u32 = 5000; /// UI Automation's "the app took too long". const TIMEOUT: windows::core::HRESULT = windows::core::HRESULT(0x8013_1505_u32 as i32); /// "No scroll" for one direction of `SetScrollPercent`. @@ -28,10 +25,6 @@ const NO_SCROLL: f64 = -1.0; pub fn act(id: u64, target: &[i32], action: &Action) -> Result { let session = Session::start()?; - // SAFETY: a COM call on this session's interface. - if let Ok(quick) = session.automation.cast::() { - let _ = unsafe { quick.SetTransactionTimeout(TRANSACTION_MS) }; - } let (element, control) = session.find(id, target)?; if control.password { return Err(DesktopError::Password); diff --git a/crates/botloftd/src/platform/desktop/windows/act_tests.rs b/crates/botloftd/src/platform/desktop/windows/act_tests.rs index 31026e1a..a579ba6e 100644 --- a/crates/botloftd/src/platform/desktop/windows/act_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/act_tests.rs @@ -22,6 +22,17 @@ fn with_value(window: &TestWindow, value: &str) -> Control { .unwrap_or_else(|| panic!("no control with the value {value:?}")) } +/// Whether `done` comes true within two seconds. +fn soon(done: impl Fn() -> bool) -> bool { + for _ in 0..20 { + if done() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + done() +} + fn title(window: &TestWindow) -> String { listed() .expect("windows") @@ -84,12 +95,19 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .expect("the combo box"); let large = Action::Select("Large".to_owned()); act(window.id, &size.runtime_id, &large).expect("choose"); - let chosen = read(window.id) - .expect("read") - .into_iter() - .find(|control| control.kind == "combo box") - .expect("the combo box"); - assert_eq!(chosen.value.as_deref(), Some("Large")); + // The window takes the choice a moment later, as the tool waits for. + let chosen = || { + read(window.id) + .ok()? + .into_iter() + .find(|control| control.kind == "combo box") + .and_then(|control| control.value) + }; + assert!( + soon(|| chosen().as_deref() == Some("Large")), + "{:?}", + chosen() + ); let none = Action::Select("Huge".to_owned()); assert!(matches!( act(window.id, &size.runtime_id, &none), @@ -110,7 +128,7 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .find(|control| control.kind == "list") .expect("the list"); act(window.id, &list.runtime_id, &Action::Scroll(Scroll::Bottom)).expect("scroll"); - assert!(shown(&last), "scrolled to its end"); + assert!(soon(|| shown(&last)), "scrolled to its end"); } #[test] diff --git a/crates/botloftd/src/platform/desktop/windows/uia.rs b/crates/botloftd/src/platform/desktop/windows/uia.rs index 03519b26..08ebb59e 100644 --- a/crates/botloftd/src/platform/desktop/windows/uia.rs +++ b/crates/botloftd/src/platform/desktop/windows/uia.rs @@ -5,14 +5,15 @@ //! control a reading named, to act on it. use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; use windows::Win32::System::Com::{ CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, }; use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationCondition, - IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, + CUIAutomation, IUIAutomation, IUIAutomation2, IUIAutomationCacheRequest, + IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_IsPasswordPropertyId, UIA_IsValuePatternAvailablePropertyId, UIA_NamePropertyId, UIA_PROPERTY_ID, UIA_RuntimeIdPropertyId, @@ -24,11 +25,16 @@ use super::super::DesktopError; use super::super::read::Control; use super::handle; use super::uia_control::{control, kind_of}; +use windows::core::Interface; -/// The most controls one walk gathers, and how deep it goes: a huge -/// window is cut short, not read for minutes. +/// The most controls one walk gathers, how deep it goes and how long it +/// takes: a huge or stuck window is cut short, not read for minutes. const CONTROLS_MAX: usize = 3000; const DEPTH_MAX: usize = 40; +const WALK_MAX: Duration = Duration::from_secs(15); +/// How long one call into an app may take, and connecting to it: an app +/// busy with a dialog, or stuck, would otherwise hold the call for long. +const CALL_MS: u32 = 5000; const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ UIA_NamePropertyId, @@ -95,6 +101,10 @@ impl Session { let automation: IUIAutomation = CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) .map_err(|err| system(&err))?; + if let Ok(limited) = automation.cast::() { + let _ = limited.SetTransactionTimeout(CALL_MS); + let _ = limited.SetConnectionTimeout(CALL_MS); + } let cache = automation .CreateCacheRequest() .map_err(|err| system(&err))?; @@ -138,6 +148,7 @@ impl Session { target, hit: None, found: Vec::new(), + until: Instant::now() + WALK_MAX, }; // SAFETY: as above. unsafe { walk.children(&root, 0) }; @@ -167,6 +178,8 @@ struct Walk<'a> { target: Option<&'a [i32]>, hit: Option, found: Vec, + /// When the walk gives up and keeps what it has. + until: Instant, } /// The controls of window `id`, in the order they sit in it. @@ -177,7 +190,11 @@ pub fn read(id: u64) -> Result, DesktopError> { impl Walk<'_> { /// Adds the shown children of `element`, and theirs. unsafe fn children(&mut self, element: &IUIAutomationElement, depth: usize) { - if depth > DEPTH_MAX || self.found.len() >= CONTROLS_MAX || self.hit.is_some() { + if depth > DEPTH_MAX + || self.found.len() >= CONTROLS_MAX + || self.hit.is_some() + || Instant::now() > self.until + { return; } let session = self.session; diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index 08464846..62b04788 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -7,7 +7,7 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use botloft_core::ids::BotId; -use botloft_core::protocol::DesktopLevel; +use botloft_core::protocol::{DesktopAction, DesktopLevel}; use serde_json::{Value, json}; use tracing::debug; @@ -216,7 +216,7 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul } /// The bot read or acted in `window`: its panel shows it (spec 24.9). -pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { let now = daemon.clock.now_ms(); let state = daemon.desktop.activity.used(bot, window, action, now); daemon.emit(Event::DesktopChanged(state)); diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index cf1cec8c..ed43eab9 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -6,7 +6,7 @@ use std::time::Duration; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use botloft_core::protocol::{DesktopAction, DesktopActionKind, DesktopGrant, DesktopLevel}; use super::desktop::used; use super::desktop_grant::{blocking, describe, granted, heading}; @@ -40,6 +40,21 @@ fn done(action: &Action, control: &Control) -> String { } } +/// The action as the bot's panel shows it. +fn shown(action: &Action, control: &Control) -> DesktopAction { + let (kind, option) = match action { + Action::Click => (DesktopActionKind::Click, None), + Action::Type(_) => (DesktopActionKind::Type, None), + Action::Select(option) => (DesktopActionKind::Select, Some(option.clone())), + Action::Scroll(_) => (DesktopActionKind::Scroll, None), + }; + DesktopAction { + kind, + target: control.name.clone(), + option, + } +} + pub(super) async fn act( daemon: &Daemon, bot: &BotId, @@ -92,7 +107,7 @@ pub(super) async fn act( let (text, next) = render(&controls, 0); daemon.desktop.keep(bot, id, controls); let mut answer = done(&ask.action, &control); - used(daemon, bot, &window, Some(answer.clone())); + used(daemon, bot, &window, Some(shown(&ask.action, &control))); if acted == Acted::Waiting { answer.push_str( " The app did not answer in time: it may be showing a dialog. Call desktop_windows \ diff --git a/docs/spec.md b/docs/spec.md index 2c64c4a2..273c2c37 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1713,7 +1713,7 @@ O bot lê e usa as janelas pela **UI Automation** do Windows (`IUIAutomation`, C - escolher numa lista: o item pelo nome (`FindFirst` nos descendentes) com `SelectionItem.Select`, quando ele aparece sem abrir nada. Numa caixa de opções, a lista só existe aberta, e a clássica do Win32 só aceita a escolha pela ação padrão do item na lista aberta (`LegacyIAccessible.DoDefaultAction`, um clique duplo), que também a fecha; fechá-la de outro jeito cancela a escolha, como Esc (visto no Windows 11). A lista aberta fecha se a janela perde o foco no meio: aí a tool diz que não achou a opção, e o bot tenta de novo; - rolar: `Scroll` (uma página para baixo ou para cima, ou `SetScrollPercent` ao topo ou ao fim), ou, num item de lista, `ScrollItem.ScrollIntoView`. - Antes de agir, o controle é achado de novo pelo `RuntimeId` guardado na leitura, numa descida que para ao achá-lo; sumiu, a tool diz para ler de novo. Controle desativado não é usado. -- Cada chamada a um app tem 5 s (`IUIAutomation2.TransactionTimeout`): um clique que abre um diálogo pode só voltar quando ele fecha. Passado esse tempo, a ação vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. +- Cada chamada a um app tem 5 s, e conectar-se a ele também (`IUIAutomation2.TransactionTimeout` e `ConnectionTimeout`, em toda sessão, de leitura também): um clique que abre um diálogo pode só voltar quando ele fecha, e um app ocupado ou travado seguraria a chamada por muito tempo. Passado esse tempo numa ação, ela vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. A leitura de uma janela inteira para em 15 s e fica com o que leu. Visto no Windows 11: logo depois de uma caixa de opções escolhida pela ação padrão, a janela pode demorar a responder; sem esses limites, a leitura seguinte ficou presa por minutos. - Uma sessão de UI Automation por vez no processo do daemon: duas, em threads diferentes ao mesmo tempo, falham com "erro não especificado" (visto no Windows 11). - Sem o padrão que a ação pede, a tool explica o que faltou. Com mouse e teclado de verdade ligados (24.7), ela faz a ação com eles; sem, diz que precisaria deles. - Depois de agir, a tool espera 400 ms e devolve a janela lida de novo, com o título de agora (um clique pode mudá-lo), como o navegador. Ela abre com o que fez: `Clicked button "Save".`, `Typed in edit.`, `Chose "Large" in combo box.`, `Scrolled list.`. @@ -1756,7 +1756,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App - **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. -- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação como a tool disse (`Clicked button "Save".`; nada depois de só ler), quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. - **Parar tudo:** o atalho global `Ctrl+Alt+End`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele.