From 617de7591aca802cf7d71880cd9e5658ea1cb078 Mon Sep 17 00:00:00 2001 From: Phelipe Lorran <112786693+httpsphl@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:18:06 -0300 Subject: [PATCH] feat(desktop): the bot's cursor on the owner's screen The owner asked to see the bot's cursor on their own screen, not only in the panel. The outline's window now draws it over the window in use: an arrow in the bot's color with its name in a pill beside it, gliding to the point of each action, a growing ring where it clicks and dots while it types. The owner's mouse never moves for it. Reading leaves it where it was; another window starts without one until the bot acts there. --- crates/botloftd/src/desktop/outline.rs | 141 +++++++++--- crates/botloftd/src/platform/desktop/mod.rs | 2 +- .../botloftd/src/platform/desktop/overlay.rs | 27 ++- .../src/platform/desktop/windows/mod.rs | 1 + .../src/platform/desktop/windows/outline.rs | 76 +++++-- .../desktop/windows/outline_cursor.rs | 210 ++++++++++++++++++ .../platform/desktop/windows/outline_tests.rs | 8 +- docs/spec.md | 2 +- 8 files changed, 409 insertions(+), 58 deletions(-) create mode 100644 crates/botloftd/src/platform/desktop/windows/outline_cursor.rs diff --git a/crates/botloftd/src/desktop/outline.rs b/crates/botloftd/src/desktop/outline.rs index e736592b..dc78a8dc 100644 --- a/crates/botloftd/src/desktop/outline.rs +++ b/crates/botloftd/src/desktop/outline.rs @@ -2,18 +2,19 @@ //! 24.9): it shows when a bot reads or acts in a window and follows the //! window while the bot's turn goes on, until the bot leaves the desktop //! alone for a minute, its turn ends or the owner stops it. One window at -//! a time: the one used last. +//! a time: the one used last. The bot's cursor goes with it, at the point +//! of its last action in that window. use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use botloft_core::ids::BotId; -use botloft_core::protocol::{BotState, DesktopState}; +use botloft_core::protocol::{BotState, DesktopActionKind, DesktopState}; use tokio::sync::broadcast::error::RecvError; use super::notice::color; -use crate::platform::desktop; +use crate::platform::desktop::{self, ScreenCursor}; use crate::state::{Daemon, Event}; /// How often the outline catches up with its window. @@ -29,11 +30,17 @@ struct Outlined { bot: BotId, window: u64, color: (u8, u8, u8), + name: String, + /// Where the bot acted last in the window. + cursor: Option, used: Instant, /// When the bot's turn ended, if it did. turn_over: Option, } +/// What the outline shows now: the window, the bot's color and cursor. +type Shown = (u64, (u8, u8, u8), Option); + /// What the outline does, apart from the screen. #[derive(Debug, Default)] struct Follow { @@ -44,11 +51,12 @@ struct Follow { } impl Follow { - /// A bot read or acted in a window, or the owner stopped it. + /// A bot read or acted in a window, or the owner stopped it; `who` + /// gives the bot's color and name. fn changed( &mut self, state: &DesktopState, - color: impl FnOnce() -> (u8, u8, u8), + who: impl FnOnce() -> ((u8, u8, u8), String), now: Instant, ) { let mine = self @@ -68,14 +76,34 @@ impl Follow { if self.last.insert(state.bot_id.clone(), at) == Some(at) { return; } - let color = match &self.outlined { - Some(outlined) if mine => outlined.color, - _ => color(), + let (color, name, before) = match self.outlined.take() { + Some(outlined) if mine => { + let before = outlined.cursor.filter(|_| outlined.window == window.id); + (outlined.color, outlined.name, before) + } + _ => { + let (color, name) = who(); + (color, name, None) + } + }; + // Only reading leaves the cursor where it was. + let cursor = match &state.action { + Some(action) => action.x.zip(action.y).map(|(x, y)| ScreenCursor { + x, + y, + click: action.kind == DesktopActionKind::Click, + typing: action.kind == DesktopActionKind::Type, + name: name.clone(), + at, + }), + None => before, }; self.outlined = Some(Outlined { bot: state.bot_id.clone(), window: window.id, color, + name, + cursor, used: now, turn_over: None, }); @@ -93,7 +121,7 @@ impl Follow { } /// The window to outline now, if any. - fn now(&mut self, now: Instant) -> Option<(u64, (u8, u8, u8))> { + fn now(&mut self, now: Instant) -> Option { let outlined = self.outlined.as_ref()?; let over = outlined .turn_over @@ -103,7 +131,7 @@ impl Follow { self.outlined = None; return None; } - Some((outlined.window, outlined.color)) + Some((outlined.window, outlined.color, outlined.cursor.clone())) } } @@ -121,8 +149,8 @@ pub async fn run(daemon: Arc) { loop { tokio::select! { _ = tick.tick() => match follow.now(Instant::now()) { - Some((window, color)) => { - desktop::outline_show(window, color); + Some((window, color, cursor)) => { + desktop::outline_show(window, color, cursor); shown = true; } None if shown => { @@ -133,11 +161,17 @@ pub async fn run(daemon: Arc) { }, event = events.recv() => match event { Ok(Event::DesktopChanged(state)) => { - let hex = || { + let who = || { let bot = daemon.store().bot(&state.bot_id).ok().flatten(); - color(&bot.map(|bot| bot.color).unwrap_or_default()) + let (hex, name) = bot.map(|bot| (bot.color, bot.name)).unwrap_or_default(); + (color(&hex), name) }; - follow.changed(&state, hex, Instant::now()); + follow.changed(&state, who, Instant::now()); + // The cursor sets off now, not at the next tick. + if let Some((window, color, cursor)) = follow.now(Instant::now()) { + desktop::outline_show(window, color, cursor); + shown = true; + } } Ok(Event::BotState(change)) => { follow.turn(&change.bot_id, working(change.state), Instant::now()); @@ -151,7 +185,7 @@ pub async fn run(daemon: Arc) { #[cfg(test)] mod tests { - use botloft_core::protocol::DesktopWindow; + use botloft_core::protocol::{DesktopAction, DesktopWindow}; use super::*; @@ -171,47 +205,90 @@ mod tests { const BLUE: (u8, u8, u8) = (0x5E, 0xC8, 0xFF); + /// The window and color outlined at `now`, without the cursor. + fn spot(follow: &mut Follow, now: Instant) -> Option<(u64, (u8, u8, u8))> { + follow.now(now).map(|(window, color, _)| (window, color)) + } + + #[test] + fn the_cursor_is_at_the_last_action_and_stays_while_the_bot_reads() { + let mut follow = Follow::default(); + let (scout, start) = (BotId::generate(), Instant::now()); + let mut clicked = used(&scout, 7, 1); + clicked.action = Some(DesktopAction { + kind: DesktopActionKind::Click, + target: "Save".to_owned(), + option: None, + x: Some(0.25), + y: Some(0.5), + }); + follow.changed(&clicked, || (BLUE, "Scout".to_owned()), start); + let cursor = follow + .now(start) + .and_then(|(_, _, cursor)| cursor) + .expect("cursor"); + assert_eq!( + (cursor.x, cursor.y, cursor.click, cursor.typing), + (0.25, 0.5, true, false) + ); + assert_eq!((cursor.name.as_str(), cursor.at), ("Scout", 1)); + + follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start); + let kept = follow.now(start).and_then(|(_, _, cursor)| cursor); + assert_eq!(kept.map(|cursor| cursor.at), Some(1), "reading leaves it"); + // Another window starts without one. + follow.changed(&used(&scout, 8, 3), || (BLUE, "Scout".to_owned()), start); + assert_eq!(follow.now(start).and_then(|(_, _, cursor)| cursor), None); + } + #[test] fn the_outline_follows_the_window_in_use_until_the_turn_ends() { let mut follow = Follow::default(); let (scout, start) = (BotId::generate(), Instant::now()); - assert_eq!(follow.now(start), None); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - assert_eq!(follow.now(start), Some((7, BLUE))); + assert_eq!(spot(&mut follow, start), None); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), Some((7, BLUE))); // Its turn ends: a moment more, then gone. follow.turn(&scout, false, start); - assert_eq!(follow.now(start + Duration::from_secs(1)), Some((7, BLUE))); - assert_eq!(follow.now(start + AFTER_TURN), None); + assert_eq!( + spot(&mut follow, start + Duration::from_secs(1)), + Some((7, BLUE)) + ); + assert_eq!(spot(&mut follow, start + AFTER_TURN), None); } #[test] fn a_minute_away_from_the_desktop_or_a_stop_takes_it_away() { let mut follow = Follow::default(); let (scout, start) = (BotId::generate(), Instant::now()); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - assert_eq!(follow.now(start + IDLE_FOR), None); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start + IDLE_FOR), None); - follow.changed(&used(&scout, 7, 2), || BLUE, start); + follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start); let mut stopped = used(&scout, 7, 2); stopped.stopped = true; - follow.changed(&stopped, || BLUE, start); - assert_eq!(follow.now(start), None); + follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), None); // Letting it go on is not a use. stopped.stopped = false; - follow.changed(&stopped, || BLUE, start); - assert_eq!(follow.now(start), None); + follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), None); } #[test] fn the_window_used_last_is_the_one_outlined() { let mut follow = Follow::default(); let (scout, writer, start) = (BotId::generate(), BotId::generate(), Instant::now()); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - follow.changed(&used(&writer, 9, 2), || (1, 2, 3), start); - assert_eq!(follow.now(start), Some((9, (1, 2, 3)))); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + follow.changed( + &used(&writer, 9, 2), + || ((1, 2, 3), "Writer".to_owned()), + start, + ); + assert_eq!(spot(&mut follow, start), Some((9, (1, 2, 3)))); // Another bot's turn ending leaves it. follow.turn(&scout, false, start); - assert_eq!(follow.now(start + AFTER_TURN), Some((9, (1, 2, 3)))); + assert_eq!(spot(&mut follow, start + AFTER_TURN), Some((9, (1, 2, 3)))); } } diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index f853bae1..7cd6c548 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -16,7 +16,7 @@ mod windows; use std::path::PathBuf; pub use blocked::{Never, never}; -pub use overlay::{notice_hide, notice_show, outline_hide, outline_show}; +pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show}; pub use read::{Control, LINES_MAX, by_reference, reference, render}; /// The largest picture of a window the bot gets, in logical pixels (24.4). diff --git a/crates/botloftd/src/platform/desktop/overlay.rs b/crates/botloftd/src/platform/desktop/overlay.rs index 8fb4d0ae..57dc3a56 100644 --- a/crates/botloftd/src/platform/desktop/overlay.rs +++ b/crates/botloftd/src/platform/desktop/overlay.rs @@ -5,6 +5,22 @@ #[cfg(windows)] use super::windows; +/// The bot's cursor on the owner's screen, over the window it uses. +#[derive(Debug, Clone, PartialEq)] +pub struct ScreenCursor { + /// Where, as fractions of the window's frame from its top left corner. + pub x: f64, + pub y: f64, + /// A ring shows where it clicked. + pub click: bool, + /// Dots show while it types. + pub typing: bool, + /// The bot's name, beside the arrow. + pub name: String, + /// When the bot acted: a new one glides there. + pub at: i64, +} + /// Shows the notice that a bot uses the real mouse and keyboard: `text` /// in a pill, a border in `color` around the monitor of window `over` /// (spec 24.7). @@ -21,13 +37,14 @@ pub fn notice_hide() { windows::notice_hide(); } -/// Outlines window `over` in `color`, or moves the outline to where the -/// window is now; hidden while the window is minimized or gone. -pub fn outline_show(over: u64, color: (u8, u8, u8)) { +/// Outlines window `over` in `color`, with the bot's `cursor` in it, or +/// moves the outline to where the window is now; hidden while the window +/// is minimized or gone. +pub fn outline_show(over: u64, color: (u8, u8, u8), cursor: Option) { #[cfg(windows)] - windows::outline_show(over, color); + windows::outline_show(over, color, cursor); #[cfg(not(windows))] - let _ = (over, color); + let _ = (over, color, cursor); } /// Takes the outline away. diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index deb1c554..b740f640 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -9,6 +9,7 @@ mod capture; mod input; mod notice; mod outline; +mod outline_cursor; #[cfg(test)] mod outline_tests; mod owner; diff --git a/crates/botloftd/src/platform/desktop/windows/outline.rs b/crates/botloftd/src/platform/desktop/windows/outline.rs index 58a4dd1f..77047df8 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline.rs @@ -5,6 +5,7 @@ //! top of everything: a window of the owner's over the app covers the //! outline too. Never in the list of windows (a tool window), never taking //! the focus, never in a picture of the app (`PrintWindow` is only of it). +//! The bot's cursor is drawn in it too (`outline_cursor.rs`). use std::sync::mpsc::{self, Sender}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -19,14 +20,21 @@ use windows::Win32::UI::HiDpi::{ }; use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GW_HWNDPREV, GetClientRect, GetMessageW, - GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, LWA_COLORKEY, MSG, PostMessageW, - RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, SetLayeredWindowAttributes, - SetWindowPos, ShowWindow, TranslateMessage, WM_APP, WM_PAINT, WNDCLASSW, WS_EX_LAYERED, - WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TRANSPARENT, WS_POPUP, + GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, KillTimer, LWA_COLORKEY, MSG, + PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, + SetLayeredWindowAttributes, SetTimer, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, + WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, + WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; -use super::{capture, handle}; +use super::super::ScreenCursor; +use super::{capture, handle, outline_cursor}; + +/// The timer that draws the cursor while it moves. +const MOVING: usize = 1; +/// How often it draws then, about 60 times a second. +const FRAME_MS: u32 = 16; /// Painted, then left out: the window's see-through color. const SEE_THROUGH: COLORREF = COLORREF(0x00FF_00FF); @@ -35,11 +43,12 @@ const HIDE: u32 = WM_APP + 2; /// The ring's width, at 96 DPI. const RING: i32 = 3; -/// Which window, in what color. -#[derive(Clone, Copy, PartialEq, Eq)] +/// Which window, in what color, with the bot's cursor where. +#[derive(Clone, PartialEq)] struct Outlining { over: u64, color: (u8, u8, u8), + cursor: Option, } static OUTLINING: Mutex> = Mutex::new(None); @@ -120,13 +129,29 @@ unsafe extern "system" fn procedure( unsafe { match message { SHOW => { - let outlining = *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner); + let outlining = OUTLINING + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); match outlining { - Some(outlining) => place(hwnd, outlining), + Some(outlining) => { + place(hwnd, &outlining); + if outline_cursor::follow(outlining.cursor.as_ref()) { + let _ = InvalidateRect(Some(hwnd), None, false); + SetTimer(Some(hwnd), MOVING, FRAME_MS, None); + } + } None => hidden(hwnd), } LRESULT(0) } + WM_TIMER => { + let _ = InvalidateRect(Some(hwnd), None, false); + if !outline_cursor::moving(std::time::Instant::now()) { + let _ = KillTimer(Some(hwnd), MOVING); + } + LRESULT(0) + } HIDE => { hidden(hwnd); LRESULT(0) @@ -142,13 +167,14 @@ unsafe extern "system" fn procedure( unsafe fn hidden(hwnd: HWND) { *PLACED.lock().unwrap_or_else(PoisonError::into_inner) = None; + outline_cursor::follow(None); // SAFETY: hides this thread's own window. let _ = unsafe { ShowWindow(hwnd, SW_HIDE) }; } /// Around the app's frame, right above it in the order of windows; hidden /// while the app is minimized, hidden or gone. -unsafe fn place(hwnd: HWND, outlining: Outlining) { +unsafe fn place(hwnd: HWND, outlining: &Outlining) { let app = handle(outlining.over); // SAFETY: plain queries on a window handle. let showing = unsafe { @@ -188,9 +214,12 @@ unsafe fn place(hwnd: HWND, outlining: Outlining) { } unsafe fn paint(hwnd: HWND) { - let color = OUTLINING + let outlining = OUTLINING .lock() .unwrap_or_else(PoisonError::into_inner) + .clone(); + let color = outlining + .as_ref() .map_or((0xFF, 0x7A, 0x59), |outlining| outlining.color); // SAFETY: painting this thread's own window between Begin and EndPaint, // every object made here freed before the end. @@ -223,16 +252,33 @@ unsafe fn paint(hwnd: HWND) { ] { FillRect(dc, &edge, brush); } + if let Some(cursor) = outlining + .as_ref() + .and_then(|outlining| outlining.cursor.as_ref()) + { + let dpi = GetDpiForWindow(hwnd).max(96) as i32; + let inside = RECT { + left: area.left + ring, + top: area.top + ring, + right: area.right - ring, + bottom: area.bottom - ring, + }; + outline_cursor::paint(dc, inside, |length| length * dpi / 96, color, cursor); + } let _ = DeleteObject(brush.into()); let _ = DeleteObject(clear.into()); let _ = EndPaint(hwnd, &paint); } } -/// Outlines window `over` in `color`, or puts the outline where the window -/// is now. -pub fn show(over: u64, color: (u8, u8, u8)) { - *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = Some(Outlining { over, color }); +/// Outlines window `over` in `color`, with the bot's `cursor`, or puts the +/// outline where the window is now. +pub fn show(over: u64, color: (u8, u8, u8), cursor: Option) { + *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = Some(Outlining { + over, + color, + cursor, + }); if let Some(hwnd) = window() { // SAFETY: asks the outline's own thread to place it. let _ = unsafe { PostMessageW(Some(hwnd), SHOW, WPARAM(0), LPARAM(0)) }; diff --git a/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs b/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs new file mode 100644 index 00000000..e01bfa6f --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs @@ -0,0 +1,210 @@ +//! The bot's cursor on the owner's screen (spec 24.9), drawn in the +//! outline's window: an arrow in the bot's color with its name beside it, +//! gliding to the point of each action, a ring where it clicks and dots +//! while it types. The owner's own mouse never moves for it. + +use std::sync::{Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use windows::Win32::Foundation::{COLORREF, POINT, RECT}; +use windows::Win32::Graphics::Gdi::{ + CreateFontW, CreatePen, CreateSolidBrush, DT_CENTER, DT_SINGLELINE, DT_VCENTER, DeleteObject, + DrawTextW, Ellipse, GetStockObject, HDC, HGDIOBJ, NONANTIALIASED_QUALITY, NULL_BRUSH, PS_SOLID, + Polygon, RoundRect, SelectObject, SetBkMode, SetTextColor, TRANSPARENT, +}; +use windows::core::w; + +use super::super::ScreenCursor; + +/// How long the cursor takes to glide to a new point. +const GLIDE: Duration = Duration::from_millis(350); +/// How long the ring of a click grows. +const RIPPLE: Duration = Duration::from_millis(500); +/// The arrow, pointing at (0, 0), at 96 DPI. +const ARROW: [(i32, i32); 7] = [ + (0, 0), + (0, 17), + (4, 13), + (7, 20), + (10, 19), + (7, 12), + (12, 12), +]; + +/// Where the cursor goes and where it came from, in fractions of the +/// window. +struct Motion { + from: (f64, f64), + to: (f64, f64), + started: Instant, + /// The action it shows, by when the bot did it. + at: i64, + click: bool, +} + +static MOTION: Mutex> = Mutex::new(None); + +fn ease((from, to): ((f64, f64), (f64, f64)), done: f64) -> (f64, f64) { + let done = 1.0 - (1.0 - done.clamp(0.0, 1.0)).powi(3); + ( + from.0 + (to.0 - from.0) * done, + from.1 + (to.1 - from.1) * done, + ) +} + +fn now_at(motion: &Motion, now: Instant) -> (f64, f64) { + let done = now.duration_since(motion.started).as_secs_f64() / GLIDE.as_secs_f64(); + ease((motion.from, motion.to), done) +} + +/// Takes in the cursor to show; `true` while it moves or its ring grows, +/// for the window to draw again soon. +pub(super) fn follow(cursor: Option<&ScreenCursor>) -> bool { + let mut motion = MOTION.lock().unwrap_or_else(PoisonError::into_inner); + let Some(cursor) = cursor else { + *motion = None; + return false; + }; + let now = Instant::now(); + if motion.as_ref().is_none_or(|motion| motion.at != cursor.at) { + let to = (cursor.x, cursor.y); + let from = motion.as_ref().map_or(to, |motion| now_at(motion, now)); + *motion = Some(Motion { + from, + to, + started: now, + at: cursor.at, + click: cursor.click, + }); + } + moving(now) +} + +/// Whether the cursor still glides or its ring still grows. +pub(super) fn moving(now: Instant) -> bool { + MOTION + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .is_some_and(|motion| now.duration_since(motion.started) < GLIDE + RIPPLE) +} + +fn colorref((red, green, blue): (u8, u8, u8)) -> COLORREF { + COLORREF(u32::from(red) | (u32::from(green) << 8) | (u32::from(blue) << 16)) +} + +/// Draws the cursor in `inside`, the app's frame within the window. +/// +/// # Safety +/// `dc` is the outline window's, between `BeginPaint` and `EndPaint`. +pub(super) unsafe fn paint( + dc: HDC, + inside: RECT, + scale: impl Fn(i32) -> i32, + color: (u8, u8, u8), + cursor: &ScreenCursor, +) { + let now = Instant::now(); + let (point, ripple) = { + let motion = MOTION.lock().unwrap_or_else(PoisonError::into_inner); + let Some(motion) = motion.as_ref() else { + return; + }; + let since = now.duration_since(motion.started); + let ripple = (motion.click && since >= GLIDE && since < GLIDE + RIPPLE) + .then(|| (since - GLIDE).as_secs_f64() / RIPPLE.as_secs_f64()); + (now_at(motion, now), ripple) + }; + let x = inside.left + (point.0 * f64::from(inside.right - inside.left)) as i32; + let y = inside.top + (point.1 * f64::from(inside.bottom - inside.top)) as i32; + // SAFETY: drawing on the caller's paint DC; every object made here is + // selected out and freed before the end. + unsafe { + let fill = CreateSolidBrush(colorref(color)); + let white = CreatePen(PS_SOLID, scale(1).max(1), COLORREF(0x00FF_FFFF)); + let old_brush: HGDIOBJ = SelectObject(dc, fill.into()); + let old_pen: HGDIOBJ = SelectObject(dc, white.into()); + if let Some(done) = ripple { + let ring = CreatePen(PS_SOLID, scale(3), colorref(color)); + SelectObject(dc, ring.into()); + SelectObject(dc, GetStockObject(NULL_BRUSH)); + let radius = scale(8) + (f64::from(scale(15)) * done) as i32; + let _ = Ellipse(dc, x - radius, y - radius, x + radius, y + radius); + SelectObject(dc, white.into()); + SelectObject(dc, fill.into()); + let _ = DeleteObject(ring.into()); + } + let arrow: Vec = ARROW + .iter() + .map(|&(dx, dy)| POINT { + x: x + scale(dx), + y: y + scale(dy), + }) + .collect(); + let _ = Polygon(dc, &arrow); + // The bot's name in a pill beside it, dots while it types. + let mut label: Vec = if cursor.typing { + format!("{} \u{2022}\u{2022}\u{2022}", cursor.name) + } else { + cursor.name.clone() + } + .encode_utf16() + .collect(); + let width = scale(14) + scale(7) * label.len().min(40) as i32; + let (left, top) = (x + scale(14), y + scale(16)); + let pill = scale(20); + let _ = RoundRect(dc, left, top, left + width, top + pill, pill, pill); + let font = CreateFontW( + -scale(12), + 0, + 0, + 0, + 600, + 0, + 0, + 0, + Default::default(), + Default::default(), + Default::default(), + NONANTIALIASED_QUALITY, + 0, + w!("Segoe UI"), + ); + let old_font = SelectObject(dc, font.into()); + SetBkMode(dc, TRANSPARENT); + SetTextColor(dc, COLORREF(0x00FF_FFFF)); + let mut place = RECT { + left, + top, + right: left + width, + bottom: top + pill, + }; + DrawTextW( + dc, + &mut label, + &mut place, + DT_CENTER | DT_VCENTER | DT_SINGLELINE, + ); + SelectObject(dc, old_font); + SelectObject(dc, old_pen); + SelectObject(dc, old_brush); + for object in [font.into(), white.into(), fill.into()] { + let _ = DeleteObject(object); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_cursor_glides_and_lands_where_the_bot_acted() { + let start = ((0.0, 0.0), (1.0, 0.5)); + assert_eq!(ease(start, 0.0), (0.0, 0.0)); + assert_eq!(ease(start, 1.0), (1.0, 0.5)); + assert_eq!(ease(start, 2.0), (1.0, 0.5), "and stays"); + let (x, _) = ease(start, 0.5); + assert!(x > 0.5, "fast first, slow at the end: {x}"); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/outline_tests.rs b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs index 4d5f8a66..b3eaa473 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs @@ -55,7 +55,7 @@ fn around(frame: [i32; 4], rect: RECT) -> bool { fn the_outline_goes_around_the_window_and_follows_it() { let window = TestWindow::open("Botloft desktop test: outline"); let app = handle(window.id); - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("shown", visible); let frame = capture::frame(window.id).expect("frame"); soon("around the frame", || around(frame, place())); @@ -77,15 +77,15 @@ fn the_outline_goes_around_the_window_and_follows_it() { ) }; let moved = capture::frame(window.id).expect("frame"); - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("moved along", || around(moved, place())); // SAFETY: minimizes and restores the test's own window. let _ = unsafe { ShowWindow(app, SW_MINIMIZE) }; - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("out of the way", || !visible()); let _ = unsafe { ShowWindow(app, SW_RESTORE) }; - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("back", visible); outline::hide(); diff --git a/docs/spec.md b/docs/spec.md index 26acf413..3471e603 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1768,7 +1768,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App - **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. Sobre a foto, o cursor do bot desliza até o ponto de cada ação e um anel marca o clique, como no navegador (21.8). O painel abre sozinho quando o bot começa a usar o desktop, isto é, no primeiro `desktop.changed` com janela depois de 2 minutos sem nenhum, sobre o painel que estiver aberto, menos com o navegador nas mãos do dono, e só se o dono não desligou "seguir o bot" (15.1); fechado no meio, as ações seguintes da mesma vez não o abrem de novo. -- **Contorno na tela:** enquanto um bot lê ou age numa janela, ela ganha um contorno na cor do bot, na tela do dono (3 px a 96 DPI, em volta da moldura do DWM). É uma janela própria do daemon, num thread com fila própria: em camadas com uma cor que vira transparente, que deixa os cliques passarem, não pega o foco e não entra na lista de janelas (`WS_EX_TOOLWINDOW`). Não fica sobre tudo: vai logo acima da janela do app na ordem das janelas (atrás da que estava acima dela), então uma janela do dono na frente do app cobre o contorno também. Ela segue a janela a cada 250 ms (posição e ordem) e some com a janela minimizada, escondida ou fechada. Aparece no `desktop.changed` de uma leitura, foto ou ação, e sai 3 s depois de o turno do bot acabar, com um minuto sem uso do desktop no mesmo turno, ou quando o dono o para. Um contorno por vez: o da janela usada por último. Quem decide é uma tarefa do daemon que escuta `desktop.changed` e `bot.state` (`desktop/outline.rs`); a janela fica em `platform/desktop/windows/outline.rs`. +- **Contorno na tela:** enquanto um bot lê ou age numa janela, ela ganha um contorno na cor do bot, na tela do dono (3 px a 96 DPI, em volta da moldura do DWM). É uma janela própria do daemon, num thread com fila própria: em camadas com uma cor que vira transparente, que deixa os cliques passarem, não pega o foco e não entra na lista de janelas (`WS_EX_TOOLWINDOW`). Não fica sobre tudo: vai logo acima da janela do app na ordem das janelas (atrás da que estava acima dela), então uma janela do dono na frente do app cobre o contorno também. Ela segue a janela a cada 250 ms (posição e ordem) e some com a janela minimizada, escondida ou fechada. Aparece no `desktop.changed` de uma leitura, foto ou ação, e sai 3 s depois de o turno do bot acabar, com um minuto sem uso do desktop no mesmo turno, ou quando o dono o para. Um contorno por vez: o da janela usada por último. Dentro dele vai o **cursor do bot** na tela do dono, sem mexer no mouse dele: uma seta na cor do bot, contornada de branco, com o nome do bot numa pílula ao lado (e "•••" enquanto ele escreve). Ele desliza em 350 ms até o ponto de cada ação (`x` e `y` do `DesktopAction`), e um anel na cor do bot cresce onde ele clicou. Depois de só ler, fica onde estava; noutra janela, começa sem cursor até a primeira ação. Desenhado na janela do contorno (GDI, sem suavização, por causa da cor transparente), com um timer de 16 ms só enquanto se move (`platform/desktop/windows/outline_cursor.rs`). Quem decide é uma tarefa do daemon que escuta `desktop.changed` e `bot.state` (`desktop/outline.rs`); a janela fica em `platform/desktop/windows/outline.rs`. - **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle, a opção escolhida e o ponto, `x` e `y` em frações da moldura da janela a partir do canto de cima à esquerda: o centro do controle pela acessibilidade, ou o ponto da foto num clique de `desktop_click_at`, sem ponto nas teclas ou com o controle fora da janela; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado.