diff --git a/app/src/lib/fakeDesktop.ts b/app/src/lib/fakeDesktop.ts index 4483ea44..6f0e1de1 100644 --- a/app/src/lib/fakeDesktop.ts +++ b/app/src/lib/fakeDesktop.ts @@ -1,16 +1,29 @@ -// The fake daemon's desktop grants (spec 24.2, 24.10): what each bot may -// see and do on the owner's desktop, and `desktop.*`. The bots' desktop -// tools are not played: grants come from `grant`, as the real daemon makes -// them when the owner allows a request. +// The fake daemon's desktop (spec 24): what each bot may see and do on the +// owner's desktop (spec 24.2, 24.10), what it is doing there for its panel, +// with pictures while the panel watches, and stopping it (spec 24.9). The +// bots' desktop tools are not played: grants come from `grant` and use +// from `use`, as the real daemon makes them. import type { FakeBotloft, Handlers } from "./fake"; import { notFound } from "./fakeRules"; -import type { BotId, DesktopGrant, DesktopLevel } from "./protocol.gen"; +import type { + BotId, + DesktopAction, + DesktopFrame, + DesktopGrant, + DesktopLevel, + DesktopState, + DesktopWindow, +} from "./protocol.gen"; type DesktopMethods = Extract; export class FakeDesktop { readonly grants: DesktopGrant[] = []; + private readonly states = new Map(); + private readonly frames = new Map(); + /** The bot whose panel the app watches, if any. */ + watching: BotId | null = null; constructor(private readonly fake: FakeBotloft) {} @@ -48,6 +61,33 @@ export class FakeDesktop { this.fake.emit({ name: "bot.desktop", params: { botId, grants: this.of(botId) } }); } + state(botId: BotId): DesktopState { + return ( + this.states.get(botId) ?? { botId, window: null, action: null, at: null, stopped: false } + ); + } + + private set(botId: BotId, change: Partial): DesktopState { + const state = { ...this.state(botId), ...change }; + this.states.set(botId, state); + this.fake.emit({ name: "desktop.changed", params: state }); + return state; + } + + /** The bot read or acted in `window`; `action` says what it did. */ + use(botId: BotId, window: DesktopWindow, action: DesktopAction | null = null): DesktopState { + return this.set(botId, { window, action, at: this.fake.now }); + } + + /** A new picture of the bot's window: sent while its panel watches. */ + paint(botId: BotId, data: string, width = 800, height = 600): void { + const frame = { botId, data, width, height }; + this.frames.set(botId, frame); + if (this.watching === botId) { + this.fake.emit({ name: "desktop.frame", params: frame }); + } + } + handlers(): Pick { return { "desktop.grants": ({ botId }) => { @@ -64,6 +104,23 @@ export class FakeDesktop { this.changed(grant.botId); return { botId: grant.botId, grants: this.of(grant.botId) }; }, + "desktop.watch": ({ botId }) => { + this.fake.bot(botId); + this.watching = botId; + return { state: this.state(botId), frame: this.frames.get(botId) ?? null }; + }, + "desktop.unwatch": () => { + this.watching = null; + return null; + }, + "desktop.stop": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: true }); + }, + "desktop.resume": ({ botId }) => { + this.fake.bot(botId); + return this.set(botId, { stopped: false }); + }, }; } } diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts index 739d0b57..a54e9090 100644 --- a/app/src/lib/protocol.gen.ts +++ b/app/src/lib/protocol.gen.ts @@ -869,6 +869,74 @@ export type BotDesktop = { botId: BotId, */ grants: Array, }; +/** + * The window a bot is using on the owner's desktop. + */ +export type DesktopWindow = { id: number, title: string, +/** + * The app's name for people. + */ +app: string, }; + +/** + * What a bot did in a window (spec 24.5), for its panel, which words it + * in the owner's language. + */ +export type DesktopActionKind = "click" | "type" | "select" | "scroll"; + +export type DesktopAction = { kind: DesktopActionKind, +/** + * The control's name as the window reads it; empty when it has none. + */ +target: string, +/** + * The option chosen, for `select`. + */ +option: string | null, }; + +/** + * What a bot does on the owner's desktop, for its panel (spec 24.9): the + * params of `desktop.changed`. + */ +export type DesktopState = { botId: BotId, +/** + * The window it read or acted in last. + */ +window: DesktopWindow | null, +/** + * What it did last; `null` after only reading. + */ +action: DesktopAction | null, +/** + * When it last read or acted, Unix ms. + */ +at: number | null, +/** + * The owner stopped it (spec 24.9): its desktop tools refuse until + * they let it go on. + */ +stopped: boolean, }; + +/** + * A picture of the window a bot is using, for whoever watches its panel. + */ +export type DesktopFrame = { botId: BotId, +/** + * JPEG, base64. + */ +data: string, width: number, height: number, }; + +/** + * The result of `desktop.watch`: the bot's state and, when it uses a + * window, its picture now. + */ +export type DesktopView = { state: DesktopState, frame: DesktopFrame | null, }; + +/** + * `desktop.watch`, `desktop.stop` and `desktop.resume`. + */ +export type DesktopBotParams = { botId: BotId, }; + export type ApprovalItem = { approvalId: ApprovalId, toolName: string, summary: string, /** * What the bot says the command is for, as in `ToolItem`. The bot @@ -1551,6 +1619,10 @@ export interface RpcMethods { "rules.delete": { params: RuleIdParams; result: BotRules }; "desktop.grants": { params: DesktopGrantsParams; result: Array }; "desktop.revoke": { params: DesktopGrantIdParams; result: BotDesktop }; + "desktop.watch": { params: DesktopBotParams; result: DesktopView }; + "desktop.unwatch": { params: undefined; result: null }; + "desktop.stop": { params: DesktopBotParams; result: DesktopState }; + "desktop.resume": { params: DesktopBotParams; result: DesktopState }; "routines.list": { params: RoutinesListParams; result: Array }; "routines.create": { params: RoutinesCreateParams; result: Routine }; "routines.update": { params: RoutinesUpdateParams; result: Routine }; @@ -1600,6 +1672,8 @@ export interface RpcNotifications { "reaction.changed": ReactionChanged; "bot.rules": BotRules; "bot.desktop": BotDesktop; + "desktop.changed": DesktopState; + "desktop.frame": DesktopFrame; "browser.changed": BrowserState; "browser.action": BrowserAction; "browser.frame": BrowserFrame; diff --git a/crates/botloft-core/src/protocol/desktop.rs b/crates/botloft-core/src/protocol/desktop.rs index 8be5f5a9..32f88849 100644 --- a/crates/botloft-core/src/protocol/desktop.rs +++ b/crates/botloft-core/src/protocol/desktop.rs @@ -107,3 +107,85 @@ pub struct BotDesktop { /// Oldest first. pub grants: Vec, } + +/// The window a bot is using on the owner's desktop. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopWindow { + pub id: u64, + pub title: String, + /// The app's name for people. + pub app: String, +} + +/// What a bot did in a window (spec 24.5), for its panel, which words it +/// in the owner's language. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub enum DesktopActionKind { + Click, + Type, + Select, + Scroll, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopAction { + pub kind: DesktopActionKind, + /// The control's name as the window reads it; empty when it has none. + pub target: String, + /// The option chosen, for `select`. + pub option: Option, +} + +/// What a bot does on the owner's desktop, for its panel (spec 24.9): the +/// params of `desktop.changed`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopState { + pub bot_id: BotId, + /// The window it read or acted in last. + pub window: Option, + /// What it did last; `null` after only reading. + pub action: Option, + /// When it last read or acted, Unix ms. + pub at: Option, + /// The owner stopped it (spec 24.9): its desktop tools refuse until + /// they let it go on. + pub stopped: bool, +} + +/// A picture of the window a bot is using, for whoever watches its panel. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopFrame { + pub bot_id: BotId, + /// JPEG, base64. + pub data: String, + pub width: u32, + pub height: u32, +} + +/// The result of `desktop.watch`: the bot's state and, when it uses a +/// window, its picture now. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopView { + pub state: DesktopState, + pub frame: Option, +} + +/// `desktop.watch`, `desktop.stop` and `desktop.resume`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(test, derive(ts_rs::TS))] +pub struct DesktopBotParams { + pub bot_id: BotId, +} diff --git a/crates/botloft-core/src/protocol/export_desktop.rs b/crates/botloft-core/src/protocol/export_desktop.rs index 79cd1ef1..0230b6ca 100644 --- a/crates/botloft-core/src/protocol/export_desktop.rs +++ b/crates/botloft-core/src/protocol/export_desktop.rs @@ -12,6 +12,13 @@ pub(super) fn decls(out: &mut Out) { out.decl::(); out.decl::(); out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); + out.decl::(); } pub(super) fn methods(out: &mut Out) { @@ -25,9 +32,23 @@ pub(super) fn methods(out: &mut Out) { &out.name::(), &out.name::(), ); + let bot = out.name::(); + out.method(method::DESKTOP_WATCH, &bot, &out.name::()); + out.method(method::DESKTOP_UNWATCH, "undefined", "null"); + let state = out.name::(); + out.method(method::DESKTOP_STOP, &bot, &state); + out.method(method::DESKTOP_RESUME, &bot, &state); } pub(super) fn notifications(out: &mut Out) { let desktop = out.name::(); let _ = writeln!(out.text, " \"{}\": {desktop};", notification::BOT_DESKTOP); + let state = out.name::(); + let _ = writeln!( + out.text, + " \"{}\": {state};", + notification::DESKTOP_CHANGED + ); + let frame = out.name::(); + let _ = writeln!(out.text, " \"{}\": {frame};", notification::DESKTOP_FRAME); } diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs index f1a7ecb9..cac4565e 100644 --- a/crates/botloft-core/src/protocol/mod.rs +++ b/crates/botloft-core/src/protocol/mod.rs @@ -125,6 +125,10 @@ pub mod method { pub const RULES_DELETE: &str = "rules.delete"; pub const DESKTOP_GRANTS: &str = "desktop.grants"; pub const DESKTOP_REVOKE: &str = "desktop.revoke"; + pub const DESKTOP_WATCH: &str = "desktop.watch"; + pub const DESKTOP_UNWATCH: &str = "desktop.unwatch"; + pub const DESKTOP_STOP: &str = "desktop.stop"; + pub const DESKTOP_RESUME: &str = "desktop.resume"; pub const MESSAGES_SEND: &str = "messages.send"; pub const MESSAGES_LIST: &str = "messages.list"; pub const ATTACHMENTS_READ: &str = "attachments.read"; @@ -175,6 +179,9 @@ pub mod notification { pub const BOT_CONTEXT: &str = "bot.context"; pub const BOT_RULES: &str = "bot.rules"; pub const BOT_DESKTOP: &str = "bot.desktop"; + pub const DESKTOP_CHANGED: &str = "desktop.changed"; + /// Only to the connection watching that bot's desktop panel (spec 24.9). + pub const DESKTOP_FRAME: &str = "desktop.frame"; pub const CHAT_ITEM: &str = "chat.item"; pub const CHAT_DELTA: &str = "chat.delta"; pub const MESSAGE_CREATED: &str = "message.created"; diff --git a/crates/botloftd/src/desktop/activity.rs b/crates/botloftd/src/desktop/activity.rs new file mode 100644 index 00000000..3c5d111a --- /dev/null +++ b/crates/botloftd/src/desktop/activity.rs @@ -0,0 +1,128 @@ +//! What each bot does on the owner's desktop, for its panel (spec 24.9): +//! the window it is using, what it did last, and whether the owner stopped +//! it. A stopped bot's desktop tools refuse until the owner lets it go on. + +use std::collections::HashMap; +use std::sync::{Mutex, PoisonError}; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{DesktopAction, DesktopState, DesktopWindow}; + +use crate::platform::desktop::Window; + +/// What the bot reads when the owner stopped it. +pub const STOPPED: &str = "The owner stopped you from using their desktop. Do not try again: \ + tell them in the chat what you were doing, and they let you go on when they want."; + +#[derive(Default)] +pub struct Activity { + by_bot: Mutex>, +} + +fn blank(bot: &BotId) -> DesktopState { + DesktopState { + bot_id: bot.clone(), + window: None, + action: None, + at: None, + stopped: false, + } +} + +impl Activity { + fn change(&self, bot: &BotId, how: impl FnOnce(&mut DesktopState)) -> DesktopState { + let mut all = self.by_bot.lock().unwrap_or_else(PoisonError::into_inner); + let state = all.entry(bot.clone()).or_insert_with(|| blank(bot)); + how(state); + state.clone() + } + + pub fn state(&self, bot: &BotId) -> DesktopState { + self.by_bot + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .cloned() + .unwrap_or_else(|| blank(bot)) + } + + /// The window the bot is using now, by its number. + pub fn window(&self, bot: &BotId) -> Option { + self.state(bot).window.map(|window| window.id) + } + + /// The bot read or acted in `window` at `now`; `action` says what it + /// did, when it did something. + pub fn used( + &self, + bot: &BotId, + window: &Window, + action: Option, + now: i64, + ) -> DesktopState { + self.change(bot, |state| { + state.window = Some(DesktopWindow { + id: window.id, + title: window.title.clone(), + app: window.app.name.clone(), + }); + state.action = action; + state.at = Some(now); + }) + } + + pub fn stopped(&self, bot: &BotId) -> bool { + self.state(bot).stopped + } + + /// Stops the bot, or lets it go on. + pub fn set_stopped(&self, bot: &BotId, stopped: bool) -> DesktopState { + self.change(bot, |state| state.stopped = stopped) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use super::*; + use crate::platform::desktop::App; + + #[test] + fn a_bot_shows_its_window_and_last_action_and_stays_stopped_until_it_may_go_on() { + let activity = Activity::default(); + let bot = BotId::generate(); + assert_eq!(activity.state(&bot), blank(&bot)); + let window = Window { + id: 7, + title: "Visits".to_owned(), + class: "Any".to_owned(), + app: App { + path: PathBuf::from(r"C:\Apps\clinic.exe"), + name: "Clinic".to_owned(), + }, + minimized: false, + elevated: false, + }; + let click = DesktopAction { + kind: botloft_core::protocol::DesktopActionKind::Click, + target: "Save".to_owned(), + option: None, + }; + let used = activity.used(&bot, &window, Some(click), 9); + assert_eq!(activity.window(&bot), Some(7)); + assert_eq!( + used.window.map(|window| window.app), + Some("Clinic".to_owned()) + ); + assert_eq!(used.at, Some(9)); + + assert!(!activity.stopped(&bot)); + assert!(activity.set_stopped(&bot, true).stopped); + assert!(activity.stopped(&bot)); + // Reading again does not let it go on. + activity.used(&bot, &window, None, 10); + assert!(activity.stopped(&bot)); + assert!(!activity.set_stopped(&bot, false).stopped); + } +} diff --git a/crates/botloftd/src/desktop.rs b/crates/botloftd/src/desktop/mod.rs similarity index 83% rename from crates/botloftd/src/desktop.rs rename to crates/botloftd/src/desktop/mod.rs index cd8f1b85..f1031c55 100644 --- a/crates/botloftd/src/desktop.rs +++ b/crates/botloftd/src/desktop/mod.rs @@ -1,6 +1,13 @@ //! The owner's desktop as the bots use it (spec 24): one action at a time -//! across every bot, whether the owner is there to see it, and each bot's -//! last reading, whose refs its actions name. +//! across every bot, whether the owner is there to see it, each bot's last +//! reading, whose refs its actions name, and what each bot does there, for +//! its panel, live. + +mod activity; +mod screen; + +pub use activity::{Activity, STOPPED}; +pub use screen::DesktopWatching; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -8,8 +15,10 @@ use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopView; use tokio::sync::MutexGuard; +use self::screen::Screens; use crate::platform::desktop::Control; /// How long the owner may go without touching the computer and still count @@ -34,6 +43,8 @@ pub struct Desktop { /// Held for each desktop action: the cursor is one. turn: tokio::sync::Mutex<()>, idle: Mutex, + pub activity: Arc, + screens: Arc, } /// Why the bot may not use the desktop now. @@ -68,6 +79,8 @@ impl Desktop { readings: Mutex::new(HashMap::new()), turn: tokio::sync::Mutex::new(()), idle: Mutex::new(idle), + activity: Arc::default(), + screens: Arc::default(), } } @@ -119,6 +132,19 @@ impl Desktop { .cloned() } + /// Starts watching the window the bot is using (spec 24.9). + pub fn watch(&self, bot: &BotId) -> DesktopWatching { + self.screens.watch(bot, &self.activity) + } + + /// The bot's state and the newest picture, for a panel that opens. + pub fn view(&self, bot: &BotId) -> DesktopView { + DesktopView { + state: self.activity.state(bot), + frame: self.screens.now(bot), + } + } + /// Waits for the desktop to be free; it is the caller's until dropped. pub async fn turn(&self) -> MutexGuard<'_, ()> { self.turn.lock().await diff --git a/crates/botloftd/src/desktop/screen.rs b/crates/botloftd/src/desktop/screen.rs new file mode 100644 index 00000000..5f99e161 --- /dev/null +++ b/crates/botloftd/src/desktop/screen.rs @@ -0,0 +1,100 @@ +//! The live picture of the window a bot is using, for whoever watches its +//! desktop panel (spec 24.9). While someone watches, the window is +//! pictured four times a second and a picture goes out only when it +//! changed; when the last one stops, so does the picturing. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD as BASE64; +use botloft_core::ids::BotId; +use botloft_core::protocol::DesktopFrame; +use tokio::sync::watch; + +use super::activity::Activity; +use crate::platform::desktop; + +/// How often a watched window is pictured. +const EVERY: Duration = Duration::from_millis(250); + +type Frames = watch::Sender>>; + +/// Held by the connection; dropping it stops watching. +pub struct DesktopWatching { + pub bot: BotId, + /// The newest picture; `changed()` wakes on each new one. + pub frames: watch::Receiver>>, +} + +#[derive(Default)] +pub struct Screens { + /// Each watched bot's pictures, while its picturing runs. + running: Mutex>, +} + +impl Screens { + /// Starts watching the bot's window: its picturing runs while anyone + /// watches. + pub fn watch(self: &Arc, bot: &BotId, activity: &Arc) -> DesktopWatching { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(frames) = running.get(bot) { + return DesktopWatching { + bot: bot.clone(), + frames: frames.subscribe(), + }; + } + let (frames, watching) = watch::channel(None); + running.insert(bot.clone(), frames.clone()); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let (screens, activity, bot) = (Arc::clone(self), Arc::clone(activity), bot.clone()); + runtime.spawn(async move { screens.picture(&bot, &activity, &frames).await }); + } + DesktopWatching { + bot: bot.clone(), + frames: watching, + } + } + + /// The newest picture of the bot's window, while it is watched. + pub fn now(&self, bot: &BotId) -> Option { + self.running + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(bot) + .and_then(|frames| frames.borrow().as_deref().cloned()) + } + + /// Pictures the bot's window until nobody watches. + async fn picture(&self, bot: &BotId, activity: &Activity, frames: &Frames) { + let mut last: Option = None; + loop { + { + let mut running = self.running.lock().unwrap_or_else(PoisonError::into_inner); + if frames.receiver_count() == 0 { + running.remove(bot); + return; + } + } + if let Some(id) = activity.window(bot) { + let picture = tokio::task::spawn_blocking(move || desktop::picture(id)).await; + if let Ok(Ok(picture)) = picture + && !picture.jpeg.is_empty() + { + let data = BASE64.encode(&picture.jpeg); + if last.as_deref() != Some(data.as_str()) { + last = Some(data.clone()); + frames.send_replace(Some(Arc::new(DesktopFrame { + bot_id: bot.clone(), + data, + width: picture.width, + height: picture.height, + }))); + } + } + } + tokio::time::sleep(EVERY).await; + } + } +} diff --git a/crates/botloftd/src/main.rs b/crates/botloftd/src/main.rs index e60871c2..c14c0660 100644 --- a/crates/botloftd/src/main.rs +++ b/crates/botloftd/src/main.rs @@ -22,7 +22,7 @@ use botloftd::trash::RecycleBin; use botloftd::{approvals, autostart, keep_awake, logging, routines, secrets, server}; use clap::{Parser, Subcommand}; use tokio::net::TcpListener; -use tracing::{error, info}; +use tracing::{error, info, warn}; #[derive(Parser)] #[command(name = "botloftd", version, about = "Botloft daemon")] @@ -137,6 +137,12 @@ fn run(paths: Paths, config: Config, config_path: PathBuf, scheduled: bool) -> a }); // No bot process survived the last run, so nobody waits for these. approvals::expire_all(&daemon); + // Ctrl+Alt+End stops every bot on the desktop, app open or not (spec 24.9). + let stopping = Arc::clone(&daemon); + match platform::desktop::on_stop_key(move || botloftd::service::desktop::stop_all(&stopping)) { + Ok(()) | Err(platform::desktop::DesktopError::Unavailable) => {} + Err(err) => warn!("the shortcut that stops bots on the desktop is not available: {err}"), + } let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index 3de60059..73096570 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -126,6 +126,20 @@ pub fn windows() -> Result, DesktopError> { } } +/// Calls `stop` each time the owner presses Ctrl+Alt+End, from now on +/// (spec 24.9). Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + #[cfg(windows)] + { + windows::on_stop_key(stop) + } + #[cfg(not(windows))] + { + let _ = stop; + Err(DesktopError::Unavailable) + } +} + /// How long ago the owner last used the mouse or keyboard (spec 24.8); /// `None` where that cannot be known. pub fn owner_idle() -> Option { diff --git a/crates/botloftd/src/platform/desktop/windows/act.rs b/crates/botloftd/src/platform/desktop/windows/act.rs index bbff3cb8..1928edb5 100644 --- a/crates/botloftd/src/platform/desktop/windows/act.rs +++ b/crates/botloftd/src/platform/desktop/windows/act.rs @@ -4,23 +4,20 @@ use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - ExpandCollapseState_Collapsed, IUIAutomation2, IUIAutomationElement, - IUIAutomationExpandCollapsePattern, IUIAutomationInvokePattern, - IUIAutomationLegacyIAccessiblePattern, IUIAutomationScrollItemPattern, - IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, IUIAutomationTogglePattern, - IUIAutomationValuePattern, ScrollAmount_LargeDecrement, ScrollAmount_LargeIncrement, - ScrollAmount_NoAmount, TreeScope_Descendants, UIA_ExpandCollapsePatternId, UIA_InvokePatternId, - UIA_LegacyIAccessiblePatternId, UIA_NamePropertyId, UIA_ScrollItemPatternId, - UIA_ScrollPatternId, UIA_SelectionItemPatternId, UIA_TogglePatternId, UIA_ValuePatternId, + ExpandCollapseState_Collapsed, IUIAutomationElement, IUIAutomationExpandCollapsePattern, + IUIAutomationInvokePattern, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationScrollItemPattern, IUIAutomationScrollPattern, IUIAutomationSelectionItemPattern, + IUIAutomationTogglePattern, IUIAutomationValuePattern, ScrollAmount_LargeDecrement, + ScrollAmount_LargeIncrement, ScrollAmount_NoAmount, TreeScope_Descendants, + UIA_ExpandCollapsePatternId, UIA_InvokePatternId, UIA_LegacyIAccessiblePatternId, + UIA_NamePropertyId, UIA_ScrollItemPatternId, UIA_ScrollPatternId, UIA_SelectionItemPatternId, + UIA_TogglePatternId, UIA_ValuePatternId, }; use windows::core::{BSTR, Interface}; use super::super::{Acted, Action, DesktopError, Scroll}; use super::uia::{Session, system}; -/// How long a call into the app may take: one that opens a dialog may not -/// come back until the dialog closes. -const TRANSACTION_MS: u32 = 5000; /// UI Automation's "the app took too long". const TIMEOUT: windows::core::HRESULT = windows::core::HRESULT(0x8013_1505_u32 as i32); /// "No scroll" for one direction of `SetScrollPercent`. @@ -28,10 +25,6 @@ const NO_SCROLL: f64 = -1.0; pub fn act(id: u64, target: &[i32], action: &Action) -> Result { let session = Session::start()?; - // SAFETY: a COM call on this session's interface. - if let Ok(quick) = session.automation.cast::() { - let _ = unsafe { quick.SetTransactionTimeout(TRANSACTION_MS) }; - } let (element, control) = session.find(id, target)?; if control.password { return Err(DesktopError::Password); diff --git a/crates/botloftd/src/platform/desktop/windows/act_tests.rs b/crates/botloftd/src/platform/desktop/windows/act_tests.rs index 31026e1a..a579ba6e 100644 --- a/crates/botloftd/src/platform/desktop/windows/act_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/act_tests.rs @@ -22,6 +22,17 @@ fn with_value(window: &TestWindow, value: &str) -> Control { .unwrap_or_else(|| panic!("no control with the value {value:?}")) } +/// Whether `done` comes true within two seconds. +fn soon(done: impl Fn() -> bool) -> bool { + for _ in 0..20 { + if done() { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + done() +} + fn title(window: &TestWindow) -> String { listed() .expect("windows") @@ -84,12 +95,19 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .expect("the combo box"); let large = Action::Select("Large".to_owned()); act(window.id, &size.runtime_id, &large).expect("choose"); - let chosen = read(window.id) - .expect("read") - .into_iter() - .find(|control| control.kind == "combo box") - .expect("the combo box"); - assert_eq!(chosen.value.as_deref(), Some("Large")); + // The window takes the choice a moment later, as the tool waits for. + let chosen = || { + read(window.id) + .ok()? + .into_iter() + .find(|control| control.kind == "combo box") + .and_then(|control| control.value) + }; + assert!( + soon(|| chosen().as_deref() == Some("Large")), + "{:?}", + chosen() + ); let none = Action::Select("Huge".to_owned()); assert!(matches!( act(window.id, &size.runtime_id, &none), @@ -110,7 +128,7 @@ fn an_option_is_chosen_and_a_long_list_scrolls() { .find(|control| control.kind == "list") .expect("the list"); act(window.id, &list.runtime_id, &Action::Scroll(Scroll::Bottom)).expect("scroll"); - assert!(shown(&last), "scrolled to its end"); + assert!(soon(|| shown(&last)), "scrolled to its end"); } #[test] diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 64207d3c..c7c82c76 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -8,6 +8,7 @@ mod act_tests; mod capture; mod input; mod process; +mod stop_key; #[cfg(test)] mod test_window; #[cfg(test)] @@ -31,6 +32,7 @@ use windows::core::{BOOL, w}; pub use act::act; pub use capture::capture; pub use input::owner_idle; +pub use stop_key::on_stop_key; pub use uia::read; use super::{App, DesktopError, Window}; diff --git a/crates/botloftd/src/platform/desktop/windows/stop_key.rs b/crates/botloftd/src/platform/desktop/windows/stop_key.rs new file mode 100644 index 00000000..b70cde46 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/stop_key.rs @@ -0,0 +1,52 @@ +//! The shortcut that stops every bot on the desktop, Ctrl+Alt+End (spec +//! 24.9): registered for the whole session on a thread of its own, so it +//! works with the app closed. + +use std::sync::mpsc; + +use windows::Win32::UI::Input::KeyboardAndMouse::{ + MOD_ALT, MOD_CONTROL, MOD_NOREPEAT, RegisterHotKey, VK_END, +}; +use windows::Win32::UI::WindowsAndMessaging::{GetMessageW, MSG, WM_HOTKEY}; + +use super::super::DesktopError; + +/// The shortcut's id for this thread. +const STOP: i32 = 1; + +/// Calls `stop` each time the owner presses the shortcut, from now on. +/// Refused when another program holds the shortcut. +pub fn on_stop_key(stop: impl Fn() + Send + 'static) -> Result<(), DesktopError> { + let (told, registered) = mpsc::channel(); + std::thread::Builder::new() + .name("desktop-stop-key".to_owned()) + .spawn(move || { + // SAFETY: a shortcut for this thread's own message queue. + let held = unsafe { + RegisterHotKey( + None, + STOP, + MOD_CONTROL | MOD_ALT | MOD_NOREPEAT, + u32::from(VK_END.0), + ) + }; + let ok = held.is_ok(); + let _ = told.send(held.map_err(|err| DesktopError::System(err.message()))); + if !ok { + return; + } + let mut message = MSG::default(); + // SAFETY: this thread's own queue, read until the process ends. + while unsafe { GetMessageW(&mut message, None, 0, 0) }.as_bool() { + if message.message == WM_HOTKEY && message.wParam.0 == STOP as usize { + stop(); + } + } + }) + .map_err(|err| DesktopError::System(err.to_string()))?; + registered.recv().unwrap_or_else(|_| { + Err(DesktopError::System( + "the shortcut's thread ended".to_owned(), + )) + }) +} diff --git a/crates/botloftd/src/platform/desktop/windows/uia.rs b/crates/botloftd/src/platform/desktop/windows/uia.rs index 03519b26..08ebb59e 100644 --- a/crates/botloftd/src/platform/desktop/windows/uia.rs +++ b/crates/botloftd/src/platform/desktop/windows/uia.rs @@ -5,14 +5,15 @@ //! control a reading named, to act on it. use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; use windows::Win32::System::Com::{ CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, CoCreateInstance, CoInitializeEx, CoUninitialize, }; use windows::Win32::System::Variant::VARIANT; use windows::Win32::UI::Accessibility::{ - CUIAutomation, IUIAutomation, IUIAutomationCacheRequest, IUIAutomationCondition, - IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, + CUIAutomation, IUIAutomation, IUIAutomation2, IUIAutomationCacheRequest, + IUIAutomationCondition, IUIAutomationElement, TreeScope_Children, UIA_ControlTypePropertyId, UIA_ExpandCollapseExpandCollapseStatePropertyId, UIA_IsEnabledPropertyId, UIA_IsOffscreenPropertyId, UIA_IsPasswordPropertyId, UIA_IsValuePatternAvailablePropertyId, UIA_NamePropertyId, UIA_PROPERTY_ID, UIA_RuntimeIdPropertyId, @@ -24,11 +25,16 @@ use super::super::DesktopError; use super::super::read::Control; use super::handle; use super::uia_control::{control, kind_of}; +use windows::core::Interface; -/// The most controls one walk gathers, and how deep it goes: a huge -/// window is cut short, not read for minutes. +/// The most controls one walk gathers, how deep it goes and how long it +/// takes: a huge or stuck window is cut short, not read for minutes. const CONTROLS_MAX: usize = 3000; const DEPTH_MAX: usize = 40; +const WALK_MAX: Duration = Duration::from_secs(15); +/// How long one call into an app may take, and connecting to it: an app +/// busy with a dialog, or stuck, would otherwise hold the call for long. +const CALL_MS: u32 = 5000; const PROPERTIES: [UIA_PROPERTY_ID; 11] = [ UIA_NamePropertyId, @@ -95,6 +101,10 @@ impl Session { let automation: IUIAutomation = CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) .map_err(|err| system(&err))?; + if let Ok(limited) = automation.cast::() { + let _ = limited.SetTransactionTimeout(CALL_MS); + let _ = limited.SetConnectionTimeout(CALL_MS); + } let cache = automation .CreateCacheRequest() .map_err(|err| system(&err))?; @@ -138,6 +148,7 @@ impl Session { target, hit: None, found: Vec::new(), + until: Instant::now() + WALK_MAX, }; // SAFETY: as above. unsafe { walk.children(&root, 0) }; @@ -167,6 +178,8 @@ struct Walk<'a> { target: Option<&'a [i32]>, hit: Option, found: Vec, + /// When the walk gives up and keeps what it has. + until: Instant, } /// The controls of window `id`, in the order they sit in it. @@ -177,7 +190,11 @@ pub fn read(id: u64) -> Result, DesktopError> { impl Walk<'_> { /// Adds the shown children of `element`, and theirs. unsafe fn children(&mut self, element: &IUIAutomationElement, depth: usize) { - if depth > DEPTH_MAX || self.found.len() >= CONTROLS_MAX || self.hit.is_some() { + if depth > DEPTH_MAX + || self.found.len() >= CONTROLS_MAX + || self.hit.is_some() + || Instant::now() > self.until + { return; } let session = self.session; diff --git a/crates/botloftd/src/rpc/desktop_watch.rs b/crates/botloftd/src/rpc/desktop_watch.rs new file mode 100644 index 00000000..9390c5f6 --- /dev/null +++ b/crates/botloftd/src/rpc/desktop_watch.rs @@ -0,0 +1,50 @@ +//! A connection watching one bot's desktop panel (spec 24.9): +//! `desktop.watch` and `desktop.unwatch`, and the pictures that follow. + +use botloft_core::protocol::{DesktopBotParams, error_code, method, notification}; +use serde_json::Value; + +use super::dispatch::parse; +use super::jsonrpc::{self, RpcError}; +use crate::desktop::DesktopWatching; +use crate::service::bots; +use crate::state::Daemon; + +#[derive(Default)] +pub(super) struct DesktopWatch { + current: Option, +} + +impl DesktopWatch { + pub(super) fn request( + &mut self, + daemon: &Daemon, + name: &str, + params: Option, + ) -> Result { + // Stop the one before, even if the new one is refused. + self.current = None; + if name == method::DESKTOP_UNWATCH { + return Ok(Value::Null); + } + let params: DesktopBotParams = parse(params)?; + bots::find(&daemon.store(), ¶ms.bot_id).map_err(RpcError::from)?; + self.current = Some(daemon.desktop.watch(¶ms.bot_id)); + serde_json::to_value(daemon.desktop.view(¶ms.bot_id)) + .map_err(|err| RpcError::new(error_code::INTERNAL_ERROR, err.to_string())) + } + + /// The watched window's next picture, as a notification. Never resolves + /// while nothing is watched. + pub(super) async fn next_frame(&mut self) -> Option { + let Some(watching) = self.current.as_mut() else { + return std::future::pending().await; + }; + if watching.frames.changed().await.is_err() { + self.current = None; + return None; + } + let frame = watching.frames.borrow_and_update().clone()?; + Some(jsonrpc::notification(notification::DESKTOP_FRAME, &*frame)) + } +} diff --git a/crates/botloftd/src/rpc/dispatch.rs b/crates/botloftd/src/rpc/dispatch.rs index f1c64d06..035111b5 100644 --- a/crates/botloftd/src/rpc/dispatch.rs +++ b/crates/botloftd/src/rpc/dispatch.rs @@ -51,6 +51,8 @@ pub fn dispatch(daemon: &Daemon, name: &str, params: Option) -> Result reply(rules::delete(daemon, parse(params)?)), method::DESKTOP_GRANTS => reply(desktop::grants(daemon, parse(params)?)), method::DESKTOP_REVOKE => reply(desktop::revoke(daemon, parse(params)?)), + method::DESKTOP_STOP => reply(desktop::stop(daemon, parse(params)?)), + method::DESKTOP_RESUME => reply(desktop::resume(daemon, parse(params)?)), method::MESSAGES_SEND => reply(messages::send(daemon, parse(params)?)), method::MESSAGES_LIST => reply(messages::list(daemon, parse(params)?)), method::ATTACHMENTS_READ => reply(attachments::read(daemon, parse(params)?)), diff --git a/crates/botloftd/src/rpc/mod.rs b/crates/botloftd/src/rpc/mod.rs index d92ba5cb..3085aa46 100644 --- a/crates/botloftd/src/rpc/mod.rs +++ b/crates/botloftd/src/rpc/mod.rs @@ -1,24 +1,26 @@ //! The app's connection: one WebSocket per client, JSON-RPC 2.0 over text //! frames (spec 11). The first request must be `session.hello`. +mod desktop_watch; mod dispatch; pub mod jsonrpc; +mod notify; mod watching; use std::sync::Arc; use std::time::Duration; use axum::extract::ws::{Message, WebSocket}; -use botloft_core::protocol::{ - HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method, notification, -}; +use botloft_core::protocol::{HelloParams, HelloResult, PROTOCOL_VERSION, error_code, method}; use futures_util::stream::{SplitSink, SplitStream}; use futures_util::{SinkExt, StreamExt}; use serde_json::Value; use tokio::sync::{broadcast, mpsc, watch}; use tracing::{debug, warn}; +use self::desktop_watch::DesktopWatch; use self::jsonrpc::RpcError; +use self::notify::to_notification; use self::watching::Watch; use crate::state::{Daemon, Event}; @@ -140,10 +142,11 @@ async fn run( daemon: &Arc, ) { let mut watch = Watch::default(); + let mut desktop = DesktopWatch::default(); loop { let frame = tokio::select! { message = stream.next() => match message { - Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, frames) { + Some(Ok(Message::Text(text))) => match route(daemon, &text, &mut watch, &mut desktop, frames) { Routed::Reply(frame) => frame, Routed::InOrder(request) => answer(daemon, request).await, Routed::Aside(request) => { @@ -167,6 +170,10 @@ async fn run( frames.send_replace(frame); None } + frame = desktop.next_frame() => { + frames.send_replace(frame); + None + } }; if let Some(frame) = frame && outbox.send(frame).await.is_err() @@ -201,6 +208,7 @@ fn route( daemon: &Arc, text: &str, watch: &mut Watch, + desktop: &mut DesktopWatch, frames: &watch::Sender>, ) -> Routed { let request = match jsonrpc::parse(text) { @@ -228,6 +236,11 @@ fn route( | method::BROWSER_SWITCH_TAB | method::BROWSER_OPEN | method::BROWSER_TEACH => watch.request(daemon, &request.method, request.params), + // So is watching a bot's desktop panel (spec 24.9). + method::DESKTOP_WATCH | method::DESKTOP_UNWATCH => { + frames.send_replace(None); + desktop.request(daemon, &request.method, request.params) + } // A window of its own outlives the request (spec 21.11). method::BROWSER_WINDOW => watch.window(daemon, request.params), name if ASIDE.contains(&name) => return Routed::Aside(request), @@ -271,30 +284,3 @@ fn reply(id: &Value, result: Result) -> String { Err(err) => jsonrpc::failure(id, &err), } } - -fn to_notification(event: &Event) -> String { - use jsonrpc::notification as note; - match event { - Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), - Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), - Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), - Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), - Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), - Event::BotState(state) => note(notification::BOT_STATE, state), - Event::BotContext(context) => note(notification::BOT_CONTEXT, context), - Event::BotRules(rules) => note(notification::BOT_RULES, rules), - Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), - Event::ChatItem(item) => note(notification::CHAT_ITEM, item), - Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), - Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), - Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), - Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), - Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), - Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), - Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), - Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), - Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), - Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), - Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), - } -} diff --git a/crates/botloftd/src/rpc/notify.rs b/crates/botloftd/src/rpc/notify.rs new file mode 100644 index 00000000..1c2d65f5 --- /dev/null +++ b/crates/botloftd/src/rpc/notify.rs @@ -0,0 +1,34 @@ +//! What changed, as the notification the apps get (spec 11). + +use botloft_core::protocol::notification; + +use super::jsonrpc; +use crate::state::Event; + +pub(super) fn to_notification(event: &Event) -> String { + use jsonrpc::notification as note; + match event { + Event::CrewChanged(crew) => note(notification::CREW_CHANGED, crew), + Event::CrewDeleted(crew) => note(notification::CREW_DELETED, crew), + Event::BotChanged(bot) => note(notification::BOT_CHANGED, bot), + Event::BotDeleted(bot) => note(notification::BOT_DELETED, bot), + Event::FolderRecycled(folder) => note(notification::FOLDER_RECYCLED, folder), + Event::BotState(state) => note(notification::BOT_STATE, state), + Event::BotContext(context) => note(notification::BOT_CONTEXT, context), + Event::BotRules(rules) => note(notification::BOT_RULES, rules), + Event::BotDesktop(desktop) => note(notification::BOT_DESKTOP, desktop), + Event::DesktopChanged(state) => note(notification::DESKTOP_CHANGED, state), + Event::ChatItem(item) => note(notification::CHAT_ITEM, item), + Event::ChatDelta(delta) => note(notification::CHAT_DELTA, delta), + Event::MessageCreated(message) => note(notification::MESSAGE_CREATED, message), + Event::DeliveryChanged(delivery) => note(notification::DELIVERY_CHANGED, delivery), + Event::TaskChanged(task) => note(notification::TASK_CHANGED, task), + Event::RoutineChanged(routine) => note(notification::ROUTINE_CHANGED, routine), + Event::RoutineRun(run) => note(notification::ROUTINE_RUN, run), + Event::BrowserChanged(state) => note(notification::BROWSER_CHANGED, state), + Event::BrowserAction(action) => note(notification::BROWSER_ACTION, action), + Event::ScreenDraft(draft) => note(notification::SCREEN_DRAFT, draft), + Event::QuestionChanged(question) => note(notification::QUESTION_CHANGED, question), + Event::ReactionChanged(change) => note(notification::REACTION_CHANGED, change), + } +} diff --git a/crates/botloftd/src/service/desktop.rs b/crates/botloftd/src/service/desktop.rs index 2bdc2b7c..4d44e656 100644 --- a/crates/botloftd/src/service/desktop.rs +++ b/crates/botloftd/src/service/desktop.rs @@ -1,9 +1,15 @@ //! What each bot may see and do on the owner's desktop (spec 24.2, //! 24.10): `desktop.grants` and `desktop.revoke`. Grants come from the -//! owner answering a bot's request (`tools/desktop.rs`). +//! owner answering a bot's request (`tools/desktop.rs`). And stopping a +//! bot there, or every bot at once (spec 24.9): `desktop.stop` and +//! `desktop.resume`. use botloft_core::ids::BotId; -use botloft_core::protocol::{BotDesktop, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams}; +use botloft_core::protocol::{ + BotDesktop, DesktopBotParams, DesktopGrant, DesktopGrantIdParams, DesktopGrantsParams, + DesktopState, +}; +use tracing::{info, warn}; use super::{ApiError, ApiResult}; use crate::state::{Daemon, Event}; @@ -34,3 +40,39 @@ pub(crate) fn changed(daemon: &Daemon, bot_id: &BotId) -> ApiResult daemon.emit(Event::BotDesktop(desktop.clone())); Ok(desktop) } + +/// Stops the bot on the desktop, or lets it go on. +fn set_stopped(daemon: &Daemon, bot: &BotId, stopped: bool) -> DesktopState { + let state = daemon.desktop.activity.set_stopped(bot, stopped); + daemon.emit(Event::DesktopChanged(state.clone())); + state +} + +pub fn stop(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, true)) +} + +pub fn resume(daemon: &Daemon, params: DesktopBotParams) -> ApiResult { + super::bots::find(&daemon.store(), ¶ms.bot_id)?; + Ok(set_stopped(daemon, ¶ms.bot_id, false)) +} + +/// Stops every bot on the desktop: the owner pressed the stop shortcut +/// (spec 24.9). +pub fn stop_all(daemon: &Daemon) { + let bots = match daemon.store().bots(None, false) { + Ok(bots) => bots, + Err(err) => { + warn!("could not list the bots to stop them on the desktop: {err}"); + return; + } + }; + for bot in &bots { + set_stopped(daemon, &bot.id, true); + } + info!( + bots = bots.len(), + "the owner stopped every bot on the desktop" + ); +} diff --git a/crates/botloftd/src/state.rs b/crates/botloftd/src/state.rs index b9768d08..f4578162 100644 --- a/crates/botloftd/src/state.rs +++ b/crates/botloftd/src/state.rs @@ -8,7 +8,8 @@ use std::time::Instant; use botloft_core::protocol::{ AccountUsage, Bot, BotContextChanged, BotDeleted, BotDesktop, BotRules, BotStateChanged, BrowserAction, BrowserState, ChatDelta, ChatItemChanged, Crew, CrewDeleted, Delivery, - FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, ScreenDraft, Task, + DesktopState, FolderRecycled, Message, Question, ReactionChanged, Routine, RoutineRun, + ScreenDraft, Task, }; use botloft_store::Store; use tokio::runtime::{Handle, RuntimeFlavor}; @@ -43,6 +44,7 @@ pub enum Event { BotContext(BotContextChanged), BotRules(BotRules), BotDesktop(BotDesktop), + DesktopChanged(DesktopState), ChatItem(ChatItemChanged), ChatDelta(ChatDelta), MessageCreated(Message), diff --git a/crates/botloftd/src/tools/desktop.rs b/crates/botloftd/src/tools/desktop.rs index 3867e14e..62b04788 100644 --- a/crates/botloftd/src/tools/desktop.rs +++ b/crates/botloftd/src/tools/desktop.rs @@ -7,7 +7,7 @@ use base64::Engine as _; use base64::engine::general_purpose::STANDARD as BASE64; use botloft_core::ids::BotId; -use botloft_core::protocol::DesktopLevel; +use botloft_core::protocol::{DesktopAction, DesktopLevel}; use serde_json::{Value, json}; use tracing::debug; @@ -15,8 +15,10 @@ use super::calls::explain; use super::desktop_act::{Ask, act}; use super::desktop_grant::{blocking, granted, heading}; use super::desktop_list::list; +use crate::desktop::STOPPED; +use crate::platform::desktop::Window; use crate::platform::desktop::{self as platform, Action, Scroll, render}; -use crate::state::Daemon; +use crate::state::{Daemon, Event}; pub(super) const PREFIX: &str = "desktop_"; @@ -136,6 +138,9 @@ pub(super) async fn call( } async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Result { + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } daemon .desktop .owner_here() @@ -161,10 +166,14 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let controls = blocking(move || platform::read(id)).await?; let (text, next) = render(&controls, from); daemon.desktop.keep(bot, id, controls); + used(daemon, bot, &window, None); let mut reading = format!("{}\n{text}", heading(&window)); if let Some(next) = next { reading.push_str(&format!( @@ -186,8 +195,12 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul ) .await?; let _turn = daemon.desktop.turn().await; + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let id = window.id; let picture = blocking(move || platform::picture(id)).await?; + used(daemon, bot, &window, None); if picture.jpeg.is_empty() { return Err("The picture of that window came out empty.".to_owned()); } @@ -201,3 +214,10 @@ async fn run(daemon: &Daemon, bot: &BotId, generation: u64, tool: Tool) -> Resul .map(Reply::Text), } } + +/// The bot read or acted in `window`: its panel shows it (spec 24.9). +pub(super) fn used(daemon: &Daemon, bot: &BotId, window: &Window, action: Option) { + let now = daemon.clock.now_ms(); + let state = daemon.desktop.activity.used(bot, window, action, now); + daemon.emit(Event::DesktopChanged(state)); +} diff --git a/crates/botloftd/src/tools/desktop_act.rs b/crates/botloftd/src/tools/desktop_act.rs index 29db9662..ed43eab9 100644 --- a/crates/botloftd/src/tools/desktop_act.rs +++ b/crates/botloftd/src/tools/desktop_act.rs @@ -6,9 +6,11 @@ use std::time::Duration; use botloft_core::ids::BotId; -use botloft_core::protocol::{DesktopGrant, DesktopLevel}; +use botloft_core::protocol::{DesktopAction, DesktopActionKind, DesktopGrant, DesktopLevel}; +use super::desktop::used; use super::desktop_grant::{blocking, describe, granted, heading}; +use crate::desktop::STOPPED; use crate::platform::desktop::{ self as platform, Acted, Action, Control, DesktopError, Window, by_reference, render, }; @@ -38,6 +40,21 @@ fn done(action: &Action, control: &Control) -> String { } } +/// The action as the bot's panel shows it. +fn shown(action: &Action, control: &Control) -> DesktopAction { + let (kind, option) = match action { + Action::Click => (DesktopActionKind::Click, None), + Action::Type(_) => (DesktopActionKind::Type, None), + Action::Select(option) => (DesktopActionKind::Select, Some(option.clone())), + Action::Scroll(_) => (DesktopActionKind::Scroll, None), + }; + DesktopAction { + kind, + target: control.name.clone(), + option, + } +} + pub(super) async fn act( daemon: &Daemon, bot: &BotId, @@ -73,6 +90,10 @@ pub(super) async fn act( ) .await?; let _turn = daemon.desktop.turn().await; + // The owner may have stopped it while it waited. + if daemon.desktop.activity.stopped(bot) { + return Err(STOPPED.to_owned()); + } let (id, target, action) = (window.id, control.runtime_id.clone(), ask.action.clone()); let acted = blocking(move || platform::act(id, &target, &action)).await?; tokio::time::sleep(SETTLE).await; @@ -86,6 +107,7 @@ pub(super) async fn act( let (text, next) = render(&controls, 0); daemon.desktop.keep(bot, id, controls); let mut answer = done(&ask.action, &control); + used(daemon, bot, &window, Some(shown(&ask.action, &control))); if acted == Acted::Waiting { answer.push_str( " The app did not answer in time: it may be showing a dialog. Call desktop_windows \ diff --git a/crates/botloftd/tests/common/desktop.rs b/crates/botloftd/tests/common/desktop.rs new file mode 100644 index 00000000..ca9415ec --- /dev/null +++ b/crates/botloftd/tests/common/desktop.rs @@ -0,0 +1,169 @@ +//! The desktop tests' own window (spec 24): a Save button that renames it, +//! a field and a password field, on a thread with its message loop; and a +//! daemon with a crew and a bot ready to use it. Windows only. + +use std::sync::mpsc; +use std::thread::JoinHandle; +use std::time::Duration; + +use serde_json::{Value, json}; +use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::WindowsAndMessaging::{ + BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, + GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, + TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, + WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, +}; +use windows::core::{PCWSTR, w}; + +use super::bots::ready_bot; +use super::mcp::Mcp; +use super::{Client, TestDaemon}; + +pub const TITLE: &str = "Botloft desktop: the clinic"; + +unsafe extern "system" fn procedure( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + if message == WM_DESTROY { + // SAFETY: ends this thread's message loop. + unsafe { PostQuitMessage(0) }; + return LRESULT(0); + } + // The Save button (id 1) clicked: the window says so in its title. + if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { + // SAFETY: renames our own window, on its thread. + let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; + return LRESULT(0); + } + // SAFETY: everything else as Windows does by default. + unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } +} + +/// A window of the test's own, on a thread with its message loop. +pub struct TestWindow { + pub id: u64, + thread: Option>, +} + +impl TestWindow { + pub fn open() -> Self { + let (sent, opened) = mpsc::channel(); + let thread = std::thread::spawn(move || { + // SAFETY: a window and its controls on this thread, with its loop. + unsafe { + let class = WNDCLASSW { + lpfnWndProc: Some(procedure), + hInstance: GetModuleHandleW(None).expect("module").into(), + lpszClassName: w!("BotloftDesktopToolsTest"), + ..Default::default() + }; + RegisterClassW(&class); + let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); + let hwnd = CreateWindowExW( + WINDOW_EX_STYLE::default(), + w!("BotloftDesktopToolsTest"), + PCWSTR(title.as_ptr()), + WS_OVERLAPPEDWINDOW | WS_VISIBLE, + CW_USEDEFAULT, + CW_USEDEFAULT, + 420, + 320, + None, + None, + None, + None, + ) + .expect("window"); + let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { + CreateWindowExW( + WINDOW_EX_STYLE::default(), + class, + text, + WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), + 10, + top, + 160, + 24, + Some(hwnd), + Some(HMENU(id as *mut std::ffi::c_void)), + None, + None, + ) + .expect("control"); + }; + child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); + child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); + child( + w!("EDIT"), + w!("hunter2"), + WS_BORDER.0 | ES_PASSWORD as u32, + 70, + 3, + ); + sent.send(hwnd.0 as usize as u64).expect("send"); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } + }); + let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); + std::thread::sleep(Duration::from_millis(300)); + Self { + id, + thread: Some(thread), + } + } +} + +impl Drop for TestWindow { + fn drop(&mut self) { + let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); + // SAFETY: asks the window's own thread to close it. + let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +pub struct Setup { + pub t: TestDaemon, + pub app: Client, + pub bot: Value, + pub mcp: Mcp, +} + +pub async fn setup() -> Setup { + let t = TestDaemon::start_supervised().await; + let mut app = t.session().await; + let crew = app + .call("crews.create", json!({ "name": "Clinic" })) + .await + .expect("crew"); + let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; + Setup { t, app, bot, mcp } +} + +pub fn reading(result: &Result) -> &str { + match result { + Ok(text) | Err(text) => text, + } +} + +/// The ref of the line that has `what` in a reading. +pub fn ref_in(reading: &str, what: &str) -> String { + let line = reading + .lines() + .find(|line| line.contains(what)) + .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); + let start = line.find('[').expect("a ref") + 1; + let end = line.find(']').expect("a ref"); + line[start..end].to_owned() +} diff --git a/crates/botloftd/tests/common/mod.rs b/crates/botloftd/tests/common/mod.rs index 7a9dfe84..bf81669f 100644 --- a/crates/botloftd/tests/common/mod.rs +++ b/crates/botloftd/tests/common/mod.rs @@ -6,6 +6,8 @@ pub mod bots; pub mod browsing; pub mod context; +#[cfg(windows)] +pub mod desktop; pub mod mcp; pub mod routines; pub mod site; diff --git a/crates/botloftd/tests/desktop.rs b/crates/botloftd/tests/desktop.rs index 466b3710..0e1a405f 100644 --- a/crates/botloftd/tests/desktop.rs +++ b/crates/botloftd/tests/desktop.rs @@ -6,161 +6,11 @@ mod common; -use std::sync::mpsc; -use std::thread::JoinHandle; use std::time::Duration; -use common::Client; -use common::TestDaemon; -use common::bots::ready_bot; use common::browsing::{call, pending_approval}; -use common::mcp::Mcp; -use serde_json::{Value, json}; -use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, WPARAM}; -use windows::Win32::System::LibraryLoader::GetModuleHandleW; -use windows::Win32::UI::WindowsAndMessaging::{ - BS_PUSHBUTTON, CW_USEDEFAULT, CreateWindowExW, DefWindowProcW, DispatchMessageW, ES_PASSWORD, - GetMessageW, HMENU, MSG, PostMessageW, PostQuitMessage, RegisterClassW, SetWindowTextW, - TranslateMessage, WINDOW_EX_STYLE, WINDOW_STYLE, WM_CLOSE, WM_COMMAND, WM_DESTROY, WNDCLASSW, - WS_BORDER, WS_CHILD, WS_OVERLAPPEDWINDOW, WS_VISIBLE, -}; -use windows::core::{PCWSTR, w}; - -const TITLE: &str = "Botloft desktop: the clinic"; - -unsafe extern "system" fn procedure( - hwnd: HWND, - message: u32, - wparam: WPARAM, - lparam: LPARAM, -) -> LRESULT { - if message == WM_DESTROY { - // SAFETY: ends this thread's message loop. - unsafe { PostQuitMessage(0) }; - return LRESULT(0); - } - // The Save button (id 1) clicked: the window says so in its title. - if message == WM_COMMAND && wparam.0 & 0xFFFF == 1 && wparam.0 >> 16 == 0 { - // SAFETY: renames our own window, on its thread. - let _ = unsafe { SetWindowTextW(hwnd, w!("Saved")) }; - return LRESULT(0); - } - // SAFETY: everything else as Windows does by default. - unsafe { DefWindowProcW(hwnd, message, wparam, lparam) } -} - -/// A window of the test's own, on a thread with its message loop. -struct TestWindow { - id: u64, - thread: Option>, -} - -impl TestWindow { - fn open() -> Self { - let (sent, opened) = mpsc::channel(); - let thread = std::thread::spawn(move || { - // SAFETY: a window and its controls on this thread, with its loop. - unsafe { - let class = WNDCLASSW { - lpfnWndProc: Some(procedure), - hInstance: GetModuleHandleW(None).expect("module").into(), - lpszClassName: w!("BotloftDesktopToolsTest"), - ..Default::default() - }; - RegisterClassW(&class); - let title: Vec = TITLE.encode_utf16().chain(Some(0)).collect(); - let hwnd = CreateWindowExW( - WINDOW_EX_STYLE::default(), - w!("BotloftDesktopToolsTest"), - PCWSTR(title.as_ptr()), - WS_OVERLAPPEDWINDOW | WS_VISIBLE, - CW_USEDEFAULT, - CW_USEDEFAULT, - 420, - 320, - None, - None, - None, - None, - ) - .expect("window"); - let child = |class: PCWSTR, text: PCWSTR, style: u32, top: i32, id: usize| { - CreateWindowExW( - WINDOW_EX_STYLE::default(), - class, - text, - WS_CHILD | WS_VISIBLE | WINDOW_STYLE(style), - 10, - top, - 160, - 24, - Some(hwnd), - Some(HMENU(id as *mut std::ffi::c_void)), - None, - None, - ) - .expect("control"); - }; - child(w!("BUTTON"), w!("Save"), BS_PUSHBUTTON as u32, 10, 1); - child(w!("EDIT"), w!("Ana Lima"), WS_BORDER.0, 40, 2); - child( - w!("EDIT"), - w!("hunter2"), - WS_BORDER.0 | ES_PASSWORD as u32, - 70, - 3, - ); - sent.send(hwnd.0 as usize as u64).expect("send"); - let mut message = MSG::default(); - while GetMessageW(&mut message, None, 0, 0).as_bool() { - let _ = TranslateMessage(&message); - DispatchMessageW(&message); - } - } - }); - let id = opened.recv_timeout(Duration::from_secs(5)).expect("opened"); - std::thread::sleep(Duration::from_millis(300)); - Self { - id, - thread: Some(thread), - } - } -} - -impl Drop for TestWindow { - fn drop(&mut self) { - let hwnd = HWND(self.id as usize as *mut std::ffi::c_void); - // SAFETY: asks the window's own thread to close it. - let _ = unsafe { PostMessageW(Some(hwnd), WM_CLOSE, WPARAM(0), LPARAM(0)) }; - if let Some(thread) = self.thread.take() { - let _ = thread.join(); - } - } -} - -struct Setup { - t: TestDaemon, - app: Client, - bot: Value, - mcp: Mcp, -} - -async fn setup() -> Setup { - let t = TestDaemon::start_supervised().await; - let mut app = t.session().await; - let crew = app - .call("crews.create", json!({ "name": "Clinic" })) - .await - .expect("crew"); - let (bot, _, mcp) = ready_bot(&t, &mut app, &crew, "Scout").await; - Setup { t, app, bot, mcp } -} - -fn reading(result: &Result) -> &str { - match result { - Ok(text) | Err(text) => text, - } -} +use common::desktop::{TITLE, TestWindow, reading, ref_in, setup}; +use serde_json::json; #[tokio::test(flavor = "multi_thread")] async fn a_bot_sees_an_app_once_the_owner_lets_it_and_never_a_password() { @@ -251,17 +101,6 @@ async fn nothing_is_seen_while_the_owner_is_away() { ); } -/// The ref of the line that has `what` in a reading. -fn ref_in(reading: &str, what: &str) -> String { - let line = reading - .lines() - .find(|line| line.contains(what)) - .unwrap_or_else(|| panic!("{what} is not in the reading:\n{reading}")); - let start = line.find('[').expect("a ref") + 1; - let end = line.find(']').expect("a ref"); - line[start..end].to_owned() -} - #[tokio::test(flavor = "multi_thread")] async fn a_bot_uses_an_app_once_the_owner_lets_it_and_never_types_a_password() { let window = TestWindow::open(); diff --git a/crates/botloftd/tests/desktop_panel.rs b/crates/botloftd/tests/desktop_panel.rs new file mode 100644 index 00000000..b4f2cc7b --- /dev/null +++ b/crates/botloftd/tests/desktop_panel.rs @@ -0,0 +1,83 @@ +//! A bot's desktop panel with real windows (spec 24.9): the window it is +//! using and what it did, its pictures while the panel is open, and the +//! owner stopping it, one bot or all of them. Windows only. +#![cfg(windows)] + +mod common; + +use common::browsing::{call, pending_approval}; +use common::desktop::{TITLE, TestWindow, reading, setup}; +use serde_json::json; + +#[tokio::test(flavor = "multi_thread")] +async fn the_panel_shows_the_window_the_bot_uses_and_its_pictures() { + let window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + // Nothing used yet: no window, no picture. + let view = s + .app + .call("desktop.watch", id.clone()) + .await + .expect("watch"); + assert_eq!(view["state"]["window"], serde_json::Value::Null); + assert_eq!(view["state"]["stopped"], false); + assert_eq!(view["frame"], serde_json::Value::Null); + + let look = json!({ "window": window.id, "why": "To read the visit" }); + let looking = call(&s.mcp, "desktop_look", look); + let asked = pending_approval(&mut s.app).await; + let allow = json!({ "approvalId": asked["approvalId"], "allow": true }); + s.app.call("approvals.answer", allow).await.expect("answer"); + looking.await.expect("task").expect("reading"); + + // The panel hears which window, and its picture follows. + let changed = s.app.notification("desktop.changed").await; + assert_eq!(changed["window"]["title"], TITLE); + assert_eq!(changed["action"], serde_json::Value::Null); + let frame = s.app.notification("desktop.frame").await; + assert_eq!(frame["botId"], s.bot["id"]); + assert!(frame["data"].as_str().is_some_and(|data| data.len() > 100)); + + s.app + .call("desktop.unwatch", serde_json::Value::Null) + .await + .expect("unwatch"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn a_stopped_bot_cannot_use_the_desktop_until_the_owner_lets_it_go_on() { + let _window = TestWindow::open(); + let mut s = setup().await; + let id = json!({ "botId": s.bot["id"] }); + + let stopped = s.app.call("desktop.stop", id.clone()).await.expect("stop"); + assert_eq!(stopped["stopped"], true); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); + + let going = s + .app + .call("desktop.resume", id.clone()) + .await + .expect("resume"); + assert_eq!(going["stopped"], false); + s.mcp + .tool_text("desktop_windows", json!({})) + .await + .expect("windows again"); + + // The shortcut stops every bot at once. + botloftd::service::desktop::stop_all(&s.t.daemon); + let refused = s.mcp.tool_text("desktop_windows", json!({})).await; + assert!( + reading(&refused).contains("stopped you"), + "{}", + reading(&refused) + ); +} diff --git a/docs/spec.md b/docs/spec.md index 7dcb5f18..273c2c37 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1653,7 +1653,7 @@ Tabela `questions`: `id, crew_id, bot_id, chat_item_id, text, options (JSON), st ## 24. Desktop -Status: **D1 e D2 implementados** (24.12). É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. +Status: **D1 e D2 implementados; D3 em andamento** (24.12): o daemon já guarda o que cada bot faz no desktop, manda a foto ao vivo, para e deixa continuar, e tem o atalho; falta o painel no app. É o item 10 da seção 18. Decisões e riscos em `docs/adr/0002-desktop-use.md`. ### 24.1 O que é @@ -1713,7 +1713,7 @@ O bot lê e usa as janelas pela **UI Automation** do Windows (`IUIAutomation`, C - escolher numa lista: o item pelo nome (`FindFirst` nos descendentes) com `SelectionItem.Select`, quando ele aparece sem abrir nada. Numa caixa de opções, a lista só existe aberta, e a clássica do Win32 só aceita a escolha pela ação padrão do item na lista aberta (`LegacyIAccessible.DoDefaultAction`, um clique duplo), que também a fecha; fechá-la de outro jeito cancela a escolha, como Esc (visto no Windows 11). A lista aberta fecha se a janela perde o foco no meio: aí a tool diz que não achou a opção, e o bot tenta de novo; - rolar: `Scroll` (uma página para baixo ou para cima, ou `SetScrollPercent` ao topo ou ao fim), ou, num item de lista, `ScrollItem.ScrollIntoView`. - Antes de agir, o controle é achado de novo pelo `RuntimeId` guardado na leitura, numa descida que para ao achá-lo; sumiu, a tool diz para ler de novo. Controle desativado não é usado. -- Cada chamada a um app tem 5 s (`IUIAutomation2.TransactionTimeout`): um clique que abre um diálogo pode só voltar quando ele fecha. Passado esse tempo, a ação vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. +- Cada chamada a um app tem 5 s, e conectar-se a ele também (`IUIAutomation2.TransactionTimeout` e `ConnectionTimeout`, em toda sessão, de leitura também): um clique que abre um diálogo pode só voltar quando ele fecha, e um app ocupado ou travado seguraria a chamada por muito tempo. Passado esse tempo numa ação, ela vale como feita e a tool diz que o app não respondeu a tempo, talvez com um diálogo aberto. A leitura de uma janela inteira para em 15 s e fica com o que leu. Visto no Windows 11: logo depois de uma caixa de opções escolhida pela ação padrão, a janela pode demorar a responder; sem esses limites, a leitura seguinte ficou presa por minutos. - Uma sessão de UI Automation por vez no processo do daemon: duas, em threads diferentes ao mesmo tempo, falham com "erro não especificado" (visto no Windows 11). - Sem o padrão que a ação pede, a tool explica o que faltou. Com mouse e teclado de verdade ligados (24.7), ela faz a ação com eles; sem, diz que precisaria deles. - Depois de agir, a tool espera 400 ms e devolve a janela lida de novo, com o título de agora (um clique pode mudá-lo), como o navegador. Ela abre com o que fez: `Clicked button "Save".`, `Typed in edit.`, `Chose "Large" in combo box.`, `Scrolled list.`. @@ -1755,8 +1755,11 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App -- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela em que o bot está agindo (`PrintWindow` a cada mudança, no máximo 5 por segundo, só enquanto o painel está aberto), o nome do app, e a linha do que ele acabou de fazer. Um botão **Parar** encerra o que o bot faz no desktop na hora. -- **Parar tudo**: um atalho global, `Ctrl+Alt+End`, que o daemon registra (`RegisterHotKey`), para qualquer ação de desktop de qualquer bot, mesmo com o app fechado. +- **Painel "Desktop"** ao lado do chat, no mesmo dock do navegador, terminal e arquivos (15.1): a foto ao vivo da janela que o bot leu ou em que agiu por último, o nome do app e o título da janela, e a linha do que ele acabou de fazer. Um botão **Parar** para o bot no desktop; parado, o painel diz isso e oferece **Deixar continuar**. +- **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle e a opção escolhida; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. +- **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. +- **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado. +- **Parar tudo:** o atalho global `Ctrl+Alt+End`, que o daemon registra ao subir (`RegisterHotKey` num thread com fila própria, `MOD_NOREPEAT`), para todos os bots no desktop, mesmo com o app fechado. Se outro programa já tem o atalho (outro daemon do Botloft, de dev, por exemplo), o daemon avisa no log e segue sem ele. - O cartão de permissão no chat (24.2) diz " quer ver " com o ícone de tela, o porquê do bot (" diz: ..."), o caminho do executável e o que ver deixa fazer (ler as janelas do app e fotografá-las com o dono no computador, nunca senhas, e onde tirar). Respondido, vira uma linha: " pode ver ", " não pode ver " com a nota, ou sem resposta. As tools `desktop_*` aparecem no chat com o ícone de tela e o porquê como resumo. ### 24.10 Permissões no app @@ -1784,7 +1787,7 @@ No servidor `botloft` (11), como as do navegador. |---|---|---| | **D1** Ver | `desktop_grants`, permissões pelo chat (app, Ver), o que nunca é liberado, `desktop_windows`, `desktop_look`, `desktop_screenshot` de uma janela, a seção Desktop nas configurações do bot | unidade; Windows de verdade com uma janela de teste própria (Win32 com botões e campos, inclusive de senha) | | **D2** Mexer | nível Mexer, `desktop_click`, `desktop_type`, `desktop_select`, `desktop_scroll` por acessibilidade, uma ação por vez | a janela de teste e o Bloco de Notas | -| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | app com FakeBotloft; o atalho no Windows de verdade | +| **D3** Painel | o painel "Desktop" ao vivo, Parar, o atalho global | o daemon com uma janela de verdade (foto ao vivo, parar, deixar continuar, parar todos); app com FakeBotloft; o atalho apertado de verdade, à mão | | **D4** Mouse e teclado | a opção, `SendInput`, `desktop_press`, `desktop_click_at`, o dono assumindo pelo gancho, o aviso na tela | a janela de teste; o dono simulado por entrada não injetada | | **D5** Sem você | a opção, a tela de riscos, o aviso na volta, a regra dos 5 minutos, tela bloqueada | unidade; teste manual com a tela bloqueada | | **D6** Desktop inteiro | o alcance, a foto da tela com as janelas bloqueadas cobertas | a janela de teste e uma bloqueada |