diff --git a/crates/botloftd/src/desktop/mod.rs b/crates/botloftd/src/desktop/mod.rs index db92bcdd..6b1dd3ce 100644 --- a/crates/botloftd/src/desktop/mod.rs +++ b/crates/botloftd/src/desktop/mod.rs @@ -6,6 +6,7 @@ mod activity; mod away; pub mod notice; +pub mod outline; mod screen; pub use activity::{Activity, STOPPED}; diff --git a/crates/botloftd/src/desktop/outline.rs b/crates/botloftd/src/desktop/outline.rs new file mode 100644 index 00000000..e736592b --- /dev/null +++ b/crates/botloftd/src/desktop/outline.rs @@ -0,0 +1,217 @@ +//! The outline in the bot's color around the window it is using (spec +//! 24.9): it shows when a bot reads or acts in a window and follows the +//! window while the bot's turn goes on, until the bot leaves the desktop +//! alone for a minute, its turn ends or the owner stops it. One window at +//! a time: the one used last. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use botloft_core::ids::BotId; +use botloft_core::protocol::{BotState, DesktopState}; +use tokio::sync::broadcast::error::RecvError; + +use super::notice::color; +use crate::platform::desktop; +use crate::state::{Daemon, Event}; + +/// How often the outline catches up with its window. +const FOLLOW_EVERY: Duration = Duration::from_millis(250); +/// How long it stays after the bot's turn ended. +const AFTER_TURN: Duration = Duration::from_secs(3); +/// How long it stays with the bot working but away from the desktop. +const IDLE_FOR: Duration = Duration::from_secs(60); + +/// The window outlined now. +#[derive(Debug, Clone, PartialEq)] +struct Outlined { + bot: BotId, + window: u64, + color: (u8, u8, u8), + used: Instant, + /// When the bot's turn ended, if it did. + turn_over: Option, +} + +/// What the outline does, apart from the screen. +#[derive(Debug, Default)] +struct Follow { + outlined: Option, + /// Each bot's last use, by the state's `at`: a state with the same one + /// is not a use. + last: HashMap, +} + +impl Follow { + /// A bot read or acted in a window, or the owner stopped it. + fn changed( + &mut self, + state: &DesktopState, + color: impl FnOnce() -> (u8, u8, u8), + now: Instant, + ) { + let mine = self + .outlined + .as_ref() + .is_some_and(|outlined| outlined.bot == state.bot_id); + if state.stopped { + if mine { + self.outlined = None; + } + return; + } + let (Some(window), Some(at)) = (&state.window, state.at) else { + return; + }; + // Letting it go on is not a use. + if self.last.insert(state.bot_id.clone(), at) == Some(at) { + return; + } + let color = match &self.outlined { + Some(outlined) if mine => outlined.color, + _ => color(), + }; + self.outlined = Some(Outlined { + bot: state.bot_id.clone(), + window: window.id, + color, + used: now, + turn_over: None, + }); + } + + /// The bot's turn began or ended. + fn turn(&mut self, bot: &BotId, working: bool, now: Instant) { + if let Some(outlined) = self + .outlined + .as_mut() + .filter(|outlined| outlined.bot == *bot) + { + outlined.turn_over = (!working).then_some(now); + } + } + + /// The window to outline now, if any. + fn now(&mut self, now: Instant) -> Option<(u64, (u8, u8, u8))> { + let outlined = self.outlined.as_ref()?; + let over = outlined + .turn_over + .is_some_and(|ended| now.duration_since(ended) >= AFTER_TURN) + || now.duration_since(outlined.used) >= IDLE_FOR; + if over { + self.outlined = None; + return None; + } + Some((outlined.window, outlined.color)) + } +} + +fn working(state: BotState) -> bool { + matches!(state, BotState::Busy | BotState::NeedsApproval) +} + +/// Follows what the bots do on the desktop and keeps the outline there. +pub async fn run(daemon: Arc) { + let mut tick = tokio::time::interval(FOLLOW_EVERY); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut events = daemon.subscribe(); + let mut follow = Follow::default(); + let mut shown = false; + loop { + tokio::select! { + _ = tick.tick() => match follow.now(Instant::now()) { + Some((window, color)) => { + desktop::outline_show(window, color); + shown = true; + } + None if shown => { + desktop::outline_hide(); + shown = false; + } + None => {} + }, + event = events.recv() => match event { + Ok(Event::DesktopChanged(state)) => { + let hex = || { + let bot = daemon.store().bot(&state.bot_id).ok().flatten(); + color(&bot.map(|bot| bot.color).unwrap_or_default()) + }; + follow.changed(&state, hex, Instant::now()); + } + Ok(Event::BotState(change)) => { + follow.turn(&change.bot_id, working(change.state), Instant::now()); + } + Ok(_) | Err(RecvError::Lagged(_)) => {} + Err(RecvError::Closed) => return, + }, + } + } +} + +#[cfg(test)] +mod tests { + use botloft_core::protocol::DesktopWindow; + + use super::*; + + fn used(bot: &BotId, window: u64, at: i64) -> DesktopState { + DesktopState { + bot_id: bot.clone(), + window: Some(DesktopWindow { + id: window, + title: "Notes".to_owned(), + app: "Notepad".to_owned(), + }), + action: None, + at: Some(at), + stopped: false, + } + } + + const BLUE: (u8, u8, u8) = (0x5E, 0xC8, 0xFF); + + #[test] + fn the_outline_follows_the_window_in_use_until_the_turn_ends() { + let mut follow = Follow::default(); + let (scout, start) = (BotId::generate(), Instant::now()); + assert_eq!(follow.now(start), None); + follow.changed(&used(&scout, 7, 1), || BLUE, start); + assert_eq!(follow.now(start), Some((7, BLUE))); + + // Its turn ends: a moment more, then gone. + follow.turn(&scout, false, start); + assert_eq!(follow.now(start + Duration::from_secs(1)), Some((7, BLUE))); + assert_eq!(follow.now(start + AFTER_TURN), None); + } + + #[test] + fn a_minute_away_from_the_desktop_or_a_stop_takes_it_away() { + let mut follow = Follow::default(); + let (scout, start) = (BotId::generate(), Instant::now()); + follow.changed(&used(&scout, 7, 1), || BLUE, start); + assert_eq!(follow.now(start + IDLE_FOR), None); + + follow.changed(&used(&scout, 7, 2), || BLUE, start); + let mut stopped = used(&scout, 7, 2); + stopped.stopped = true; + follow.changed(&stopped, || BLUE, start); + assert_eq!(follow.now(start), None); + // Letting it go on is not a use. + stopped.stopped = false; + follow.changed(&stopped, || BLUE, start); + assert_eq!(follow.now(start), None); + } + + #[test] + fn the_window_used_last_is_the_one_outlined() { + let mut follow = Follow::default(); + let (scout, writer, start) = (BotId::generate(), BotId::generate(), Instant::now()); + follow.changed(&used(&scout, 7, 1), || BLUE, start); + follow.changed(&used(&writer, 9, 2), || (1, 2, 3), start); + assert_eq!(follow.now(start), Some((9, (1, 2, 3)))); + // Another bot's turn ending leaves it. + follow.turn(&scout, false, start); + assert_eq!(follow.now(start + AFTER_TURN), Some((9, (1, 2, 3)))); + } +} diff --git a/crates/botloftd/src/main.rs b/crates/botloftd/src/main.rs index db82f339..528c59e8 100644 --- a/crates/botloftd/src/main.rs +++ b/crates/botloftd/src/main.rs @@ -161,6 +161,8 @@ fn run(paths: Paths, config: Config, config_path: PathBuf, scheduled: bool) -> a tokio::spawn(courier::run(Arc::clone(&daemon))); tokio::spawn(routines::run(Arc::clone(&daemon))); tokio::spawn(browser::run(Arc::clone(&daemon))); + // The outline around the window a bot uses (spec 24.9). + tokio::spawn(botloftd::desktop::outline::run(Arc::clone(&daemon))); tokio::spawn(keep_awake::run( daemon.supervisor.busy_bots(), daemon.settings.keep_awake(), diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index ddb1c936..f853bae1 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -5,6 +5,7 @@ mod blocked; pub mod keys; +mod overlay; mod read; // Only Windows takes pictures; the tests run everywhere. #[cfg_attr(not(windows), allow(dead_code))] @@ -15,6 +16,7 @@ mod windows; use std::path::PathBuf; pub use blocked::{Never, never}; +pub use overlay::{notice_hide, notice_show, outline_hide, outline_show}; pub use read::{Control, LINES_MAX, by_reference, reference, render}; /// The largest picture of a window the bot gets, in logical pixels (24.4). @@ -260,22 +262,6 @@ pub fn frame(id: u64) -> Result<[i32; 4], DesktopError> { } } -/// Shows the notice that a bot uses the real mouse and keyboard: `text` -/// in a pill, a border in `color` around the monitor of window `over` -/// (spec 24.7). -pub fn notice_show(text: &str, color: (u8, u8, u8), over: u64) { - #[cfg(windows)] - windows::notice_show(text, color, over); - #[cfg(not(windows))] - let _ = (text, color, over); -} - -/// Takes the notice away. -pub fn notice_hide() { - #[cfg(windows)] - windows::notice_hide(); -} - /// A picture of the window `id`, only of it, even behind other windows, in /// logical pixels up to [`PICTURE_MAX`] (spec 24.4). pub fn picture(id: u64) -> Result { diff --git a/crates/botloftd/src/platform/desktop/overlay.rs b/crates/botloftd/src/platform/desktop/overlay.rs new file mode 100644 index 00000000..8fb4d0ae --- /dev/null +++ b/crates/botloftd/src/platform/desktop/overlay.rs @@ -0,0 +1,37 @@ +//! What the daemon draws on the owner's screen while a bot uses it: the +//! notice of the real mouse and keyboard (spec 24.7) and the outline around +//! the window the bot uses (spec 24.9). + +#[cfg(windows)] +use super::windows; + +/// Shows the notice that a bot uses the real mouse and keyboard: `text` +/// in a pill, a border in `color` around the monitor of window `over` +/// (spec 24.7). +pub fn notice_show(text: &str, color: (u8, u8, u8), over: u64) { + #[cfg(windows)] + windows::notice_show(text, color, over); + #[cfg(not(windows))] + let _ = (text, color, over); +} + +/// Takes the notice away. +pub fn notice_hide() { + #[cfg(windows)] + windows::notice_hide(); +} + +/// Outlines window `over` in `color`, or moves the outline to where the +/// window is now; hidden while the window is minimized or gone. +pub fn outline_show(over: u64, color: (u8, u8, u8)) { + #[cfg(windows)] + windows::outline_show(over, color); + #[cfg(not(windows))] + let _ = (over, color); +} + +/// Takes the outline away. +pub fn outline_hide() { + #[cfg(windows)] + windows::outline_hide(); +} diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index 42e981b1..deb1c554 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -8,6 +8,9 @@ mod act_tests; mod capture; mod input; mod notice; +mod outline; +#[cfg(test)] +mod outline_tests; mod owner; mod process; mod real; @@ -38,6 +41,7 @@ pub use act::act; pub use capture::{capture, frame}; pub use input::owner_idle; pub use notice::{hide as notice_hide, show as notice_show}; +pub use outline::{hide as outline_hide, show as outline_show}; pub use real::{click as real_click, press as real_press, type_text as real_type}; pub use stop_key::on_stop_key; pub use uia::read; diff --git a/crates/botloftd/src/platform/desktop/windows/outline.rs b/crates/botloftd/src/platform/desktop/windows/outline.rs new file mode 100644 index 00000000..58a4dd1f --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/outline.rs @@ -0,0 +1,250 @@ +//! The outline in the bot's color around the window a bot is using (spec +//! 24.9): a window of its own, on a thread of its own, the size of the +//! app's frame and a ring wider, see-through inside and clicked through. +//! It sits right above the app's window in the order of windows, not on +//! top of everything: a window of the owner's over the app covers the +//! outline too. Never in the list of windows (a tool window), never taking +//! the focus, never in a picture of the app (`PrintWindow` is only of it). + +use std::sync::mpsc::{self, Sender}; +use std::sync::{Mutex, OnceLock, PoisonError}; + +use windows::Win32::Foundation::{COLORREF, HWND, LPARAM, LRESULT, RECT, WPARAM}; +use windows::Win32::Graphics::Gdi::{ + BeginPaint, CreateSolidBrush, DeleteObject, EndPaint, FillRect, InvalidateRect, PAINTSTRUCT, +}; +use windows::Win32::System::LibraryLoader::GetModuleHandleW; +use windows::Win32::UI::HiDpi::{ + DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, GetDpiForWindow, SetThreadDpiAwarenessContext, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + CreateWindowExW, DefWindowProcW, DispatchMessageW, GW_HWNDPREV, GetClientRect, GetMessageW, + GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, LWA_COLORKEY, MSG, PostMessageW, + RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, SetLayeredWindowAttributes, + SetWindowPos, ShowWindow, TranslateMessage, WM_APP, WM_PAINT, WNDCLASSW, WS_EX_LAYERED, + WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TRANSPARENT, WS_POPUP, +}; +use windows::core::{PCWSTR, w}; + +use super::{capture, handle}; + +/// Painted, then left out: the window's see-through color. +const SEE_THROUGH: COLORREF = COLORREF(0x00FF_00FF); +const SHOW: u32 = WM_APP + 1; +const HIDE: u32 = WM_APP + 2; +/// The ring's width, at 96 DPI. +const RING: i32 = 3; + +/// Which window, in what color. +#[derive(Clone, Copy, PartialEq, Eq)] +struct Outlining { + over: u64, + color: (u8, u8, u8), +} + +static OUTLINING: Mutex> = Mutex::new(None); +/// Where it was last put and in what color, to repaint only on a change. +type Placed = ([i32; 4], (u8, u8, u8)); +static PLACED: Mutex> = Mutex::new(None); +static WINDOW: OnceLock> = OnceLock::new(); + +fn colorref((red, green, blue): (u8, u8, u8)) -> COLORREF { + COLORREF(u32::from(red) | (u32::from(green) << 8) | (u32::from(blue) << 16)) +} + +/// The outline's window, made on its thread the first time. +fn window() -> Option { + let raw = WINDOW.get_or_init(|| { + let (made, window) = mpsc::channel::>(); + let started = std::thread::Builder::new() + .name("desktop-outline".to_owned()) + .spawn(move || run(&made)); + if started.is_err() { + return None; + } + window.recv().ok().flatten() + }); + raw.map(|raw| HWND(raw as *mut std::ffi::c_void)) +} + +fn run(made: &Sender>) { + // SAFETY: a window class, a window and its message loop, on this thread. + unsafe { + SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2); + let Ok(module) = GetModuleHandleW(None) else { + let _ = made.send(None); + return; + }; + let class = WNDCLASSW { + lpfnWndProc: Some(procedure), + hInstance: module.into(), + lpszClassName: w!("BotloftDesktopOutline"), + ..Default::default() + }; + RegisterClassW(&class); + let created = CreateWindowExW( + WS_EX_LAYERED | WS_EX_TRANSPARENT | WS_EX_TOOLWINDOW | WS_EX_NOACTIVATE, + w!("BotloftDesktopOutline"), + PCWSTR::null(), + WS_POPUP, + 0, + 0, + 1, + 1, + None, + None, + None, + None, + ); + let Ok(hwnd) = created else { + let _ = made.send(None); + return; + }; + let _ = SetLayeredWindowAttributes(hwnd, SEE_THROUGH, 255, LWA_COLORKEY); + let _ = made.send(Some(hwnd.0 as isize)); + let mut message = MSG::default(); + while GetMessageW(&mut message, None, 0, 0).as_bool() { + let _ = TranslateMessage(&message); + DispatchMessageW(&message); + } + } +} + +unsafe extern "system" fn procedure( + hwnd: HWND, + message: u32, + wparam: WPARAM, + lparam: LPARAM, +) -> LRESULT { + // SAFETY: calls on this thread's own window. + unsafe { + match message { + SHOW => { + let outlining = *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner); + match outlining { + Some(outlining) => place(hwnd, outlining), + None => hidden(hwnd), + } + LRESULT(0) + } + HIDE => { + hidden(hwnd); + LRESULT(0) + } + WM_PAINT => { + paint(hwnd); + LRESULT(0) + } + _ => DefWindowProcW(hwnd, message, wparam, lparam), + } + } +} + +unsafe fn hidden(hwnd: HWND) { + *PLACED.lock().unwrap_or_else(PoisonError::into_inner) = None; + // SAFETY: hides this thread's own window. + let _ = unsafe { ShowWindow(hwnd, SW_HIDE) }; +} + +/// Around the app's frame, right above it in the order of windows; hidden +/// while the app is minimized, hidden or gone. +unsafe fn place(hwnd: HWND, outlining: Outlining) { + let app = handle(outlining.over); + // SAFETY: plain queries on a window handle. + let showing = unsafe { + IsWindow(Some(app)).as_bool() && IsWindowVisible(app).as_bool() && !IsIconic(app).as_bool() + }; + let Some([left, top, width, height]) = capture::frame(outlining.over).ok().filter(|_| showing) + else { + // SAFETY: this thread's own window. + unsafe { hidden(hwnd) }; + return; + }; + // SAFETY: plain queries and a move of this thread's own window. + unsafe { + let ring = RING * GetDpiForWindow(app).max(96) as i32 / 96; + let area = [left - ring, top - ring, width + 2 * ring, height + 2 * ring]; + // Right above the app: behind the window that is above it now. + let above = GetWindow(app, GW_HWNDPREV).ok(); + let after = match above { + Some(above) if above == hwnd => None, + Some(above) => Some(above), + None => Some(HWND_TOP), + }; + let mut placed = PLACED.lock().unwrap_or_else(PoisonError::into_inner); + let changed = *placed != Some((area, outlining.color)); + *placed = Some((area, outlining.color)); + drop(placed); + let mut flags = SWP_NOACTIVATE; + if after.is_none() { + flags |= windows::Win32::UI::WindowsAndMessaging::SWP_NOZORDER; + } + let _ = SetWindowPos(hwnd, after, area[0], area[1], area[2], area[3], flags); + if changed { + let _ = InvalidateRect(Some(hwnd), None, true); + } + let _ = ShowWindow(hwnd, SW_SHOWNOACTIVATE); + } +} + +unsafe fn paint(hwnd: HWND) { + let color = OUTLINING + .lock() + .unwrap_or_else(PoisonError::into_inner) + .map_or((0xFF, 0x7A, 0x59), |outlining| outlining.color); + // SAFETY: painting this thread's own window between Begin and EndPaint, + // every object made here freed before the end. + unsafe { + let mut paint = PAINTSTRUCT::default(); + let dc = BeginPaint(hwnd, &mut paint); + let mut area = RECT::default(); + let _ = GetClientRect(hwnd, &mut area); + let clear = CreateSolidBrush(SEE_THROUGH); + FillRect(dc, &area, clear); + let brush = CreateSolidBrush(colorref(color)); + let ring = RING * GetDpiForWindow(hwnd).max(96) as i32 / 96; + for edge in [ + RECT { + bottom: area.top + ring, + ..area + }, + RECT { + top: area.bottom - ring, + ..area + }, + RECT { + right: area.left + ring, + ..area + }, + RECT { + left: area.right - ring, + ..area + }, + ] { + FillRect(dc, &edge, brush); + } + let _ = DeleteObject(brush.into()); + let _ = DeleteObject(clear.into()); + let _ = EndPaint(hwnd, &paint); + } +} + +/// Outlines window `over` in `color`, or puts the outline where the window +/// is now. +pub fn show(over: u64, color: (u8, u8, u8)) { + *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = Some(Outlining { over, color }); + if let Some(hwnd) = window() { + // SAFETY: asks the outline's own thread to place it. + let _ = unsafe { PostMessageW(Some(hwnd), SHOW, WPARAM(0), LPARAM(0)) }; + } +} + +/// Takes the outline away. +pub fn hide() { + *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = None; + if let Some(Some(raw)) = WINDOW.get() { + let hwnd = HWND(*raw as *mut std::ffi::c_void); + // SAFETY: asks the outline's own thread to hide it. + let _ = unsafe { PostMessageW(Some(hwnd), HIDE, WPARAM(0), LPARAM(0)) }; + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/outline_tests.rs b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs new file mode 100644 index 00000000..4d5f8a66 --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs @@ -0,0 +1,93 @@ +//! With a real window: the outline goes around its frame, right above it +//! in the order of windows, follows it when it moves, steps aside while it +//! is minimized and goes when asked. It draws a window, but sends no input. + +use std::time::{Duration, Instant}; + +use windows::Win32::Foundation::{HWND, RECT}; +use windows::Win32::UI::WindowsAndMessaging::{ + FindWindowW, GW_HWNDPREV, GetWindow, GetWindowRect, IsWindowVisible, SW_MINIMIZE, SW_RESTORE, + SWP_NOACTIVATE, SWP_NOSIZE, SWP_NOZORDER, SetWindowPos, ShowWindow, +}; +use windows::core::w; + +use super::test_window::TestWindow; +use super::{capture, handle, outline}; + +fn outline_window() -> Option { + // SAFETY: a plain query. + unsafe { FindWindowW(w!("BotloftDesktopOutline"), None) }.ok() +} + +fn visible() -> bool { + // SAFETY: a plain query. + outline_window().is_some_and(|hwnd| unsafe { IsWindowVisible(hwnd) }.as_bool()) +} + +fn place() -> RECT { + let mut place = RECT::default(); + if let Some(hwnd) = outline_window() { + // SAFETY: `place` is a valid out-pointer. + let _ = unsafe { GetWindowRect(hwnd, &mut place) }; + } + place +} + +/// Waits a little for the outline's thread to catch up. +fn soon(what: &str, done: impl Fn() -> bool) { + let start = Instant::now(); + while !done() { + assert!(start.elapsed() < Duration::from_secs(3), "never: {what}"); + std::thread::sleep(Duration::from_millis(30)); + } +} + +/// The frame grown by the ring on each side. +fn around(frame: [i32; 4], rect: RECT) -> bool { + let ring = frame[0] - rect.left; + ring > 0 + && rect.top == frame[1] - ring + && rect.right == frame[0] + frame[2] + ring + && rect.bottom == frame[1] + frame[3] + ring +} + +#[test] +fn the_outline_goes_around_the_window_and_follows_it() { + let window = TestWindow::open("Botloft desktop test: outline"); + let app = handle(window.id); + outline::show(window.id, (0x5E, 0xC8, 0xFF)); + soon("shown", visible); + let frame = capture::frame(window.id).expect("frame"); + soon("around the frame", || around(frame, place())); + // SAFETY: a plain query. + let above = unsafe { GetWindow(app, GW_HWNDPREV) }.ok(); + assert_eq!(above, outline_window(), "right above the app"); + + // The window moves; the outline follows on the next show. + // SAFETY: moves the test's own window. + let _ = unsafe { + SetWindowPos( + app, + None, + frame[0] + 40, + frame[1] + 30, + 0, + 0, + SWP_NOSIZE | SWP_NOZORDER | SWP_NOACTIVATE, + ) + }; + let moved = capture::frame(window.id).expect("frame"); + outline::show(window.id, (0x5E, 0xC8, 0xFF)); + soon("moved along", || around(moved, place())); + + // SAFETY: minimizes and restores the test's own window. + let _ = unsafe { ShowWindow(app, SW_MINIMIZE) }; + outline::show(window.id, (0x5E, 0xC8, 0xFF)); + soon("out of the way", || !visible()); + let _ = unsafe { ShowWindow(app, SW_RESTORE) }; + outline::show(window.id, (0x5E, 0xC8, 0xFF)); + soon("back", visible); + + outline::hide(); + soon("hidden", || !visible()); +} diff --git a/docs/spec.md b/docs/spec.md index 6ad8e064..708a9e1e 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1765,6 +1765,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App - **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. Sobre a foto, o cursor do bot desliza até o ponto de cada ação e um anel marca o clique, como no navegador (21.8). O painel abre sozinho quando o bot começa a usar o desktop, isto é, no primeiro `desktop.changed` com janela depois de 2 minutos sem nenhum, sobre o painel que estiver aberto, menos com o navegador nas mãos do dono, e só se o dono não desligou "seguir o bot" (15.1); fechado no meio, as ações seguintes da mesma vez não o abrem de novo. +- **Contorno na tela:** enquanto um bot lê ou age numa janela, ela ganha um contorno na cor do bot, na tela do dono (3 px a 96 DPI, em volta da moldura do DWM). É uma janela própria do daemon, num thread com fila própria: em camadas com uma cor que vira transparente, que deixa os cliques passarem, não pega o foco e não entra na lista de janelas (`WS_EX_TOOLWINDOW`). Não fica sobre tudo: vai logo acima da janela do app na ordem das janelas (atrás da que estava acima dela), então uma janela do dono na frente do app cobre o contorno também. Ela segue a janela a cada 250 ms (posição e ordem) e some com a janela minimizada, escondida ou fechada. Aparece no `desktop.changed` de uma leitura, foto ou ação, e sai 3 s depois de o turno do bot acabar, com um minuto sem uso do desktop no mesmo turno, ou quando o dono o para. Um contorno por vez: o da janela usada por último. Quem decide é uma tarefa do daemon que escuta `desktop.changed` e `bot.state` (`desktop/outline.rs`); a janela fica em `platform/desktop/windows/outline.rs`. - **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle, a opção escolhida e o ponto, `x` e `y` em frações da moldura da janela a partir do canto de cima à esquerda: o centro do controle pela acessibilidade, ou o ponto da foto num clique de `desktop_click_at`, sem ponto nas teclas ou com o controle fora da janela; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado.