diff --git a/crates/botloftd/src/desktop/outline.rs b/crates/botloftd/src/desktop/outline.rs index e736592b..dc78a8dc 100644 --- a/crates/botloftd/src/desktop/outline.rs +++ b/crates/botloftd/src/desktop/outline.rs @@ -2,18 +2,19 @@ //! 24.9): it shows when a bot reads or acts in a window and follows the //! window while the bot's turn goes on, until the bot leaves the desktop //! alone for a minute, its turn ends or the owner stops it. One window at -//! a time: the one used last. +//! a time: the one used last. The bot's cursor goes with it, at the point +//! of its last action in that window. use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; use botloft_core::ids::BotId; -use botloft_core::protocol::{BotState, DesktopState}; +use botloft_core::protocol::{BotState, DesktopActionKind, DesktopState}; use tokio::sync::broadcast::error::RecvError; use super::notice::color; -use crate::platform::desktop; +use crate::platform::desktop::{self, ScreenCursor}; use crate::state::{Daemon, Event}; /// How often the outline catches up with its window. @@ -29,11 +30,17 @@ struct Outlined { bot: BotId, window: u64, color: (u8, u8, u8), + name: String, + /// Where the bot acted last in the window. + cursor: Option, used: Instant, /// When the bot's turn ended, if it did. turn_over: Option, } +/// What the outline shows now: the window, the bot's color and cursor. +type Shown = (u64, (u8, u8, u8), Option); + /// What the outline does, apart from the screen. #[derive(Debug, Default)] struct Follow { @@ -44,11 +51,12 @@ struct Follow { } impl Follow { - /// A bot read or acted in a window, or the owner stopped it. + /// A bot read or acted in a window, or the owner stopped it; `who` + /// gives the bot's color and name. fn changed( &mut self, state: &DesktopState, - color: impl FnOnce() -> (u8, u8, u8), + who: impl FnOnce() -> ((u8, u8, u8), String), now: Instant, ) { let mine = self @@ -68,14 +76,34 @@ impl Follow { if self.last.insert(state.bot_id.clone(), at) == Some(at) { return; } - let color = match &self.outlined { - Some(outlined) if mine => outlined.color, - _ => color(), + let (color, name, before) = match self.outlined.take() { + Some(outlined) if mine => { + let before = outlined.cursor.filter(|_| outlined.window == window.id); + (outlined.color, outlined.name, before) + } + _ => { + let (color, name) = who(); + (color, name, None) + } + }; + // Only reading leaves the cursor where it was. + let cursor = match &state.action { + Some(action) => action.x.zip(action.y).map(|(x, y)| ScreenCursor { + x, + y, + click: action.kind == DesktopActionKind::Click, + typing: action.kind == DesktopActionKind::Type, + name: name.clone(), + at, + }), + None => before, }; self.outlined = Some(Outlined { bot: state.bot_id.clone(), window: window.id, color, + name, + cursor, used: now, turn_over: None, }); @@ -93,7 +121,7 @@ impl Follow { } /// The window to outline now, if any. - fn now(&mut self, now: Instant) -> Option<(u64, (u8, u8, u8))> { + fn now(&mut self, now: Instant) -> Option { let outlined = self.outlined.as_ref()?; let over = outlined .turn_over @@ -103,7 +131,7 @@ impl Follow { self.outlined = None; return None; } - Some((outlined.window, outlined.color)) + Some((outlined.window, outlined.color, outlined.cursor.clone())) } } @@ -121,8 +149,8 @@ pub async fn run(daemon: Arc) { loop { tokio::select! { _ = tick.tick() => match follow.now(Instant::now()) { - Some((window, color)) => { - desktop::outline_show(window, color); + Some((window, color, cursor)) => { + desktop::outline_show(window, color, cursor); shown = true; } None if shown => { @@ -133,11 +161,17 @@ pub async fn run(daemon: Arc) { }, event = events.recv() => match event { Ok(Event::DesktopChanged(state)) => { - let hex = || { + let who = || { let bot = daemon.store().bot(&state.bot_id).ok().flatten(); - color(&bot.map(|bot| bot.color).unwrap_or_default()) + let (hex, name) = bot.map(|bot| (bot.color, bot.name)).unwrap_or_default(); + (color(&hex), name) }; - follow.changed(&state, hex, Instant::now()); + follow.changed(&state, who, Instant::now()); + // The cursor sets off now, not at the next tick. + if let Some((window, color, cursor)) = follow.now(Instant::now()) { + desktop::outline_show(window, color, cursor); + shown = true; + } } Ok(Event::BotState(change)) => { follow.turn(&change.bot_id, working(change.state), Instant::now()); @@ -151,7 +185,7 @@ pub async fn run(daemon: Arc) { #[cfg(test)] mod tests { - use botloft_core::protocol::DesktopWindow; + use botloft_core::protocol::{DesktopAction, DesktopWindow}; use super::*; @@ -171,47 +205,90 @@ mod tests { const BLUE: (u8, u8, u8) = (0x5E, 0xC8, 0xFF); + /// The window and color outlined at `now`, without the cursor. + fn spot(follow: &mut Follow, now: Instant) -> Option<(u64, (u8, u8, u8))> { + follow.now(now).map(|(window, color, _)| (window, color)) + } + + #[test] + fn the_cursor_is_at_the_last_action_and_stays_while_the_bot_reads() { + let mut follow = Follow::default(); + let (scout, start) = (BotId::generate(), Instant::now()); + let mut clicked = used(&scout, 7, 1); + clicked.action = Some(DesktopAction { + kind: DesktopActionKind::Click, + target: "Save".to_owned(), + option: None, + x: Some(0.25), + y: Some(0.5), + }); + follow.changed(&clicked, || (BLUE, "Scout".to_owned()), start); + let cursor = follow + .now(start) + .and_then(|(_, _, cursor)| cursor) + .expect("cursor"); + assert_eq!( + (cursor.x, cursor.y, cursor.click, cursor.typing), + (0.25, 0.5, true, false) + ); + assert_eq!((cursor.name.as_str(), cursor.at), ("Scout", 1)); + + follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start); + let kept = follow.now(start).and_then(|(_, _, cursor)| cursor); + assert_eq!(kept.map(|cursor| cursor.at), Some(1), "reading leaves it"); + // Another window starts without one. + follow.changed(&used(&scout, 8, 3), || (BLUE, "Scout".to_owned()), start); + assert_eq!(follow.now(start).and_then(|(_, _, cursor)| cursor), None); + } + #[test] fn the_outline_follows_the_window_in_use_until_the_turn_ends() { let mut follow = Follow::default(); let (scout, start) = (BotId::generate(), Instant::now()); - assert_eq!(follow.now(start), None); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - assert_eq!(follow.now(start), Some((7, BLUE))); + assert_eq!(spot(&mut follow, start), None); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), Some((7, BLUE))); // Its turn ends: a moment more, then gone. follow.turn(&scout, false, start); - assert_eq!(follow.now(start + Duration::from_secs(1)), Some((7, BLUE))); - assert_eq!(follow.now(start + AFTER_TURN), None); + assert_eq!( + spot(&mut follow, start + Duration::from_secs(1)), + Some((7, BLUE)) + ); + assert_eq!(spot(&mut follow, start + AFTER_TURN), None); } #[test] fn a_minute_away_from_the_desktop_or_a_stop_takes_it_away() { let mut follow = Follow::default(); let (scout, start) = (BotId::generate(), Instant::now()); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - assert_eq!(follow.now(start + IDLE_FOR), None); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start + IDLE_FOR), None); - follow.changed(&used(&scout, 7, 2), || BLUE, start); + follow.changed(&used(&scout, 7, 2), || (BLUE, "Scout".to_owned()), start); let mut stopped = used(&scout, 7, 2); stopped.stopped = true; - follow.changed(&stopped, || BLUE, start); - assert_eq!(follow.now(start), None); + follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), None); // Letting it go on is not a use. stopped.stopped = false; - follow.changed(&stopped, || BLUE, start); - assert_eq!(follow.now(start), None); + follow.changed(&stopped, || (BLUE, "Scout".to_owned()), start); + assert_eq!(spot(&mut follow, start), None); } #[test] fn the_window_used_last_is_the_one_outlined() { let mut follow = Follow::default(); let (scout, writer, start) = (BotId::generate(), BotId::generate(), Instant::now()); - follow.changed(&used(&scout, 7, 1), || BLUE, start); - follow.changed(&used(&writer, 9, 2), || (1, 2, 3), start); - assert_eq!(follow.now(start), Some((9, (1, 2, 3)))); + follow.changed(&used(&scout, 7, 1), || (BLUE, "Scout".to_owned()), start); + follow.changed( + &used(&writer, 9, 2), + || ((1, 2, 3), "Writer".to_owned()), + start, + ); + assert_eq!(spot(&mut follow, start), Some((9, (1, 2, 3)))); // Another bot's turn ending leaves it. follow.turn(&scout, false, start); - assert_eq!(follow.now(start + AFTER_TURN), Some((9, (1, 2, 3)))); + assert_eq!(spot(&mut follow, start + AFTER_TURN), Some((9, (1, 2, 3)))); } } diff --git a/crates/botloftd/src/platform/desktop/mod.rs b/crates/botloftd/src/platform/desktop/mod.rs index f853bae1..7cd6c548 100644 --- a/crates/botloftd/src/platform/desktop/mod.rs +++ b/crates/botloftd/src/platform/desktop/mod.rs @@ -16,7 +16,7 @@ mod windows; use std::path::PathBuf; pub use blocked::{Never, never}; -pub use overlay::{notice_hide, notice_show, outline_hide, outline_show}; +pub use overlay::{ScreenCursor, notice_hide, notice_show, outline_hide, outline_show}; pub use read::{Control, LINES_MAX, by_reference, reference, render}; /// The largest picture of a window the bot gets, in logical pixels (24.4). diff --git a/crates/botloftd/src/platform/desktop/overlay.rs b/crates/botloftd/src/platform/desktop/overlay.rs index 8fb4d0ae..57dc3a56 100644 --- a/crates/botloftd/src/platform/desktop/overlay.rs +++ b/crates/botloftd/src/platform/desktop/overlay.rs @@ -5,6 +5,22 @@ #[cfg(windows)] use super::windows; +/// The bot's cursor on the owner's screen, over the window it uses. +#[derive(Debug, Clone, PartialEq)] +pub struct ScreenCursor { + /// Where, as fractions of the window's frame from its top left corner. + pub x: f64, + pub y: f64, + /// A ring shows where it clicked. + pub click: bool, + /// Dots show while it types. + pub typing: bool, + /// The bot's name, beside the arrow. + pub name: String, + /// When the bot acted: a new one glides there. + pub at: i64, +} + /// Shows the notice that a bot uses the real mouse and keyboard: `text` /// in a pill, a border in `color` around the monitor of window `over` /// (spec 24.7). @@ -21,13 +37,14 @@ pub fn notice_hide() { windows::notice_hide(); } -/// Outlines window `over` in `color`, or moves the outline to where the -/// window is now; hidden while the window is minimized or gone. -pub fn outline_show(over: u64, color: (u8, u8, u8)) { +/// Outlines window `over` in `color`, with the bot's `cursor` in it, or +/// moves the outline to where the window is now; hidden while the window +/// is minimized or gone. +pub fn outline_show(over: u64, color: (u8, u8, u8), cursor: Option) { #[cfg(windows)] - windows::outline_show(over, color); + windows::outline_show(over, color, cursor); #[cfg(not(windows))] - let _ = (over, color); + let _ = (over, color, cursor); } /// Takes the outline away. diff --git a/crates/botloftd/src/platform/desktop/windows/mod.rs b/crates/botloftd/src/platform/desktop/windows/mod.rs index deb1c554..b740f640 100644 --- a/crates/botloftd/src/platform/desktop/windows/mod.rs +++ b/crates/botloftd/src/platform/desktop/windows/mod.rs @@ -9,6 +9,7 @@ mod capture; mod input; mod notice; mod outline; +mod outline_cursor; #[cfg(test)] mod outline_tests; mod owner; diff --git a/crates/botloftd/src/platform/desktop/windows/outline.rs b/crates/botloftd/src/platform/desktop/windows/outline.rs index 58a4dd1f..77047df8 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline.rs @@ -5,6 +5,7 @@ //! top of everything: a window of the owner's over the app covers the //! outline too. Never in the list of windows (a tool window), never taking //! the focus, never in a picture of the app (`PrintWindow` is only of it). +//! The bot's cursor is drawn in it too (`outline_cursor.rs`). use std::sync::mpsc::{self, Sender}; use std::sync::{Mutex, OnceLock, PoisonError}; @@ -19,14 +20,21 @@ use windows::Win32::UI::HiDpi::{ }; use windows::Win32::UI::WindowsAndMessaging::{ CreateWindowExW, DefWindowProcW, DispatchMessageW, GW_HWNDPREV, GetClientRect, GetMessageW, - GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, LWA_COLORKEY, MSG, PostMessageW, - RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, SetLayeredWindowAttributes, - SetWindowPos, ShowWindow, TranslateMessage, WM_APP, WM_PAINT, WNDCLASSW, WS_EX_LAYERED, - WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, WS_EX_TRANSPARENT, WS_POPUP, + GetWindow, HWND_TOP, IsIconic, IsWindow, IsWindowVisible, KillTimer, LWA_COLORKEY, MSG, + PostMessageW, RegisterClassW, SW_HIDE, SW_SHOWNOACTIVATE, SWP_NOACTIVATE, + SetLayeredWindowAttributes, SetTimer, SetWindowPos, ShowWindow, TranslateMessage, WM_APP, + WM_PAINT, WM_TIMER, WNDCLASSW, WS_EX_LAYERED, WS_EX_NOACTIVATE, WS_EX_TOOLWINDOW, + WS_EX_TRANSPARENT, WS_POPUP, }; use windows::core::{PCWSTR, w}; -use super::{capture, handle}; +use super::super::ScreenCursor; +use super::{capture, handle, outline_cursor}; + +/// The timer that draws the cursor while it moves. +const MOVING: usize = 1; +/// How often it draws then, about 60 times a second. +const FRAME_MS: u32 = 16; /// Painted, then left out: the window's see-through color. const SEE_THROUGH: COLORREF = COLORREF(0x00FF_00FF); @@ -35,11 +43,12 @@ const HIDE: u32 = WM_APP + 2; /// The ring's width, at 96 DPI. const RING: i32 = 3; -/// Which window, in what color. -#[derive(Clone, Copy, PartialEq, Eq)] +/// Which window, in what color, with the bot's cursor where. +#[derive(Clone, PartialEq)] struct Outlining { over: u64, color: (u8, u8, u8), + cursor: Option, } static OUTLINING: Mutex> = Mutex::new(None); @@ -120,13 +129,29 @@ unsafe extern "system" fn procedure( unsafe { match message { SHOW => { - let outlining = *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner); + let outlining = OUTLINING + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); match outlining { - Some(outlining) => place(hwnd, outlining), + Some(outlining) => { + place(hwnd, &outlining); + if outline_cursor::follow(outlining.cursor.as_ref()) { + let _ = InvalidateRect(Some(hwnd), None, false); + SetTimer(Some(hwnd), MOVING, FRAME_MS, None); + } + } None => hidden(hwnd), } LRESULT(0) } + WM_TIMER => { + let _ = InvalidateRect(Some(hwnd), None, false); + if !outline_cursor::moving(std::time::Instant::now()) { + let _ = KillTimer(Some(hwnd), MOVING); + } + LRESULT(0) + } HIDE => { hidden(hwnd); LRESULT(0) @@ -142,13 +167,14 @@ unsafe extern "system" fn procedure( unsafe fn hidden(hwnd: HWND) { *PLACED.lock().unwrap_or_else(PoisonError::into_inner) = None; + outline_cursor::follow(None); // SAFETY: hides this thread's own window. let _ = unsafe { ShowWindow(hwnd, SW_HIDE) }; } /// Around the app's frame, right above it in the order of windows; hidden /// while the app is minimized, hidden or gone. -unsafe fn place(hwnd: HWND, outlining: Outlining) { +unsafe fn place(hwnd: HWND, outlining: &Outlining) { let app = handle(outlining.over); // SAFETY: plain queries on a window handle. let showing = unsafe { @@ -188,9 +214,12 @@ unsafe fn place(hwnd: HWND, outlining: Outlining) { } unsafe fn paint(hwnd: HWND) { - let color = OUTLINING + let outlining = OUTLINING .lock() .unwrap_or_else(PoisonError::into_inner) + .clone(); + let color = outlining + .as_ref() .map_or((0xFF, 0x7A, 0x59), |outlining| outlining.color); // SAFETY: painting this thread's own window between Begin and EndPaint, // every object made here freed before the end. @@ -223,16 +252,33 @@ unsafe fn paint(hwnd: HWND) { ] { FillRect(dc, &edge, brush); } + if let Some(cursor) = outlining + .as_ref() + .and_then(|outlining| outlining.cursor.as_ref()) + { + let dpi = GetDpiForWindow(hwnd).max(96) as i32; + let inside = RECT { + left: area.left + ring, + top: area.top + ring, + right: area.right - ring, + bottom: area.bottom - ring, + }; + outline_cursor::paint(dc, inside, |length| length * dpi / 96, color, cursor); + } let _ = DeleteObject(brush.into()); let _ = DeleteObject(clear.into()); let _ = EndPaint(hwnd, &paint); } } -/// Outlines window `over` in `color`, or puts the outline where the window -/// is now. -pub fn show(over: u64, color: (u8, u8, u8)) { - *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = Some(Outlining { over, color }); +/// Outlines window `over` in `color`, with the bot's `cursor`, or puts the +/// outline where the window is now. +pub fn show(over: u64, color: (u8, u8, u8), cursor: Option) { + *OUTLINING.lock().unwrap_or_else(PoisonError::into_inner) = Some(Outlining { + over, + color, + cursor, + }); if let Some(hwnd) = window() { // SAFETY: asks the outline's own thread to place it. let _ = unsafe { PostMessageW(Some(hwnd), SHOW, WPARAM(0), LPARAM(0)) }; diff --git a/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs b/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs new file mode 100644 index 00000000..e01bfa6f --- /dev/null +++ b/crates/botloftd/src/platform/desktop/windows/outline_cursor.rs @@ -0,0 +1,210 @@ +//! The bot's cursor on the owner's screen (spec 24.9), drawn in the +//! outline's window: an arrow in the bot's color with its name beside it, +//! gliding to the point of each action, a ring where it clicks and dots +//! while it types. The owner's own mouse never moves for it. + +use std::sync::{Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use windows::Win32::Foundation::{COLORREF, POINT, RECT}; +use windows::Win32::Graphics::Gdi::{ + CreateFontW, CreatePen, CreateSolidBrush, DT_CENTER, DT_SINGLELINE, DT_VCENTER, DeleteObject, + DrawTextW, Ellipse, GetStockObject, HDC, HGDIOBJ, NONANTIALIASED_QUALITY, NULL_BRUSH, PS_SOLID, + Polygon, RoundRect, SelectObject, SetBkMode, SetTextColor, TRANSPARENT, +}; +use windows::core::w; + +use super::super::ScreenCursor; + +/// How long the cursor takes to glide to a new point. +const GLIDE: Duration = Duration::from_millis(350); +/// How long the ring of a click grows. +const RIPPLE: Duration = Duration::from_millis(500); +/// The arrow, pointing at (0, 0), at 96 DPI. +const ARROW: [(i32, i32); 7] = [ + (0, 0), + (0, 17), + (4, 13), + (7, 20), + (10, 19), + (7, 12), + (12, 12), +]; + +/// Where the cursor goes and where it came from, in fractions of the +/// window. +struct Motion { + from: (f64, f64), + to: (f64, f64), + started: Instant, + /// The action it shows, by when the bot did it. + at: i64, + click: bool, +} + +static MOTION: Mutex> = Mutex::new(None); + +fn ease((from, to): ((f64, f64), (f64, f64)), done: f64) -> (f64, f64) { + let done = 1.0 - (1.0 - done.clamp(0.0, 1.0)).powi(3); + ( + from.0 + (to.0 - from.0) * done, + from.1 + (to.1 - from.1) * done, + ) +} + +fn now_at(motion: &Motion, now: Instant) -> (f64, f64) { + let done = now.duration_since(motion.started).as_secs_f64() / GLIDE.as_secs_f64(); + ease((motion.from, motion.to), done) +} + +/// Takes in the cursor to show; `true` while it moves or its ring grows, +/// for the window to draw again soon. +pub(super) fn follow(cursor: Option<&ScreenCursor>) -> bool { + let mut motion = MOTION.lock().unwrap_or_else(PoisonError::into_inner); + let Some(cursor) = cursor else { + *motion = None; + return false; + }; + let now = Instant::now(); + if motion.as_ref().is_none_or(|motion| motion.at != cursor.at) { + let to = (cursor.x, cursor.y); + let from = motion.as_ref().map_or(to, |motion| now_at(motion, now)); + *motion = Some(Motion { + from, + to, + started: now, + at: cursor.at, + click: cursor.click, + }); + } + moving(now) +} + +/// Whether the cursor still glides or its ring still grows. +pub(super) fn moving(now: Instant) -> bool { + MOTION + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_ref() + .is_some_and(|motion| now.duration_since(motion.started) < GLIDE + RIPPLE) +} + +fn colorref((red, green, blue): (u8, u8, u8)) -> COLORREF { + COLORREF(u32::from(red) | (u32::from(green) << 8) | (u32::from(blue) << 16)) +} + +/// Draws the cursor in `inside`, the app's frame within the window. +/// +/// # Safety +/// `dc` is the outline window's, between `BeginPaint` and `EndPaint`. +pub(super) unsafe fn paint( + dc: HDC, + inside: RECT, + scale: impl Fn(i32) -> i32, + color: (u8, u8, u8), + cursor: &ScreenCursor, +) { + let now = Instant::now(); + let (point, ripple) = { + let motion = MOTION.lock().unwrap_or_else(PoisonError::into_inner); + let Some(motion) = motion.as_ref() else { + return; + }; + let since = now.duration_since(motion.started); + let ripple = (motion.click && since >= GLIDE && since < GLIDE + RIPPLE) + .then(|| (since - GLIDE).as_secs_f64() / RIPPLE.as_secs_f64()); + (now_at(motion, now), ripple) + }; + let x = inside.left + (point.0 * f64::from(inside.right - inside.left)) as i32; + let y = inside.top + (point.1 * f64::from(inside.bottom - inside.top)) as i32; + // SAFETY: drawing on the caller's paint DC; every object made here is + // selected out and freed before the end. + unsafe { + let fill = CreateSolidBrush(colorref(color)); + let white = CreatePen(PS_SOLID, scale(1).max(1), COLORREF(0x00FF_FFFF)); + let old_brush: HGDIOBJ = SelectObject(dc, fill.into()); + let old_pen: HGDIOBJ = SelectObject(dc, white.into()); + if let Some(done) = ripple { + let ring = CreatePen(PS_SOLID, scale(3), colorref(color)); + SelectObject(dc, ring.into()); + SelectObject(dc, GetStockObject(NULL_BRUSH)); + let radius = scale(8) + (f64::from(scale(15)) * done) as i32; + let _ = Ellipse(dc, x - radius, y - radius, x + radius, y + radius); + SelectObject(dc, white.into()); + SelectObject(dc, fill.into()); + let _ = DeleteObject(ring.into()); + } + let arrow: Vec = ARROW + .iter() + .map(|&(dx, dy)| POINT { + x: x + scale(dx), + y: y + scale(dy), + }) + .collect(); + let _ = Polygon(dc, &arrow); + // The bot's name in a pill beside it, dots while it types. + let mut label: Vec = if cursor.typing { + format!("{} \u{2022}\u{2022}\u{2022}", cursor.name) + } else { + cursor.name.clone() + } + .encode_utf16() + .collect(); + let width = scale(14) + scale(7) * label.len().min(40) as i32; + let (left, top) = (x + scale(14), y + scale(16)); + let pill = scale(20); + let _ = RoundRect(dc, left, top, left + width, top + pill, pill, pill); + let font = CreateFontW( + -scale(12), + 0, + 0, + 0, + 600, + 0, + 0, + 0, + Default::default(), + Default::default(), + Default::default(), + NONANTIALIASED_QUALITY, + 0, + w!("Segoe UI"), + ); + let old_font = SelectObject(dc, font.into()); + SetBkMode(dc, TRANSPARENT); + SetTextColor(dc, COLORREF(0x00FF_FFFF)); + let mut place = RECT { + left, + top, + right: left + width, + bottom: top + pill, + }; + DrawTextW( + dc, + &mut label, + &mut place, + DT_CENTER | DT_VCENTER | DT_SINGLELINE, + ); + SelectObject(dc, old_font); + SelectObject(dc, old_pen); + SelectObject(dc, old_brush); + for object in [font.into(), white.into(), fill.into()] { + let _ = DeleteObject(object); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_cursor_glides_and_lands_where_the_bot_acted() { + let start = ((0.0, 0.0), (1.0, 0.5)); + assert_eq!(ease(start, 0.0), (0.0, 0.0)); + assert_eq!(ease(start, 1.0), (1.0, 0.5)); + assert_eq!(ease(start, 2.0), (1.0, 0.5), "and stays"); + let (x, _) = ease(start, 0.5); + assert!(x > 0.5, "fast first, slow at the end: {x}"); + } +} diff --git a/crates/botloftd/src/platform/desktop/windows/outline_tests.rs b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs index 4d5f8a66..b3eaa473 100644 --- a/crates/botloftd/src/platform/desktop/windows/outline_tests.rs +++ b/crates/botloftd/src/platform/desktop/windows/outline_tests.rs @@ -55,7 +55,7 @@ fn around(frame: [i32; 4], rect: RECT) -> bool { fn the_outline_goes_around_the_window_and_follows_it() { let window = TestWindow::open("Botloft desktop test: outline"); let app = handle(window.id); - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("shown", visible); let frame = capture::frame(window.id).expect("frame"); soon("around the frame", || around(frame, place())); @@ -77,15 +77,15 @@ fn the_outline_goes_around_the_window_and_follows_it() { ) }; let moved = capture::frame(window.id).expect("frame"); - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("moved along", || around(moved, place())); // SAFETY: minimizes and restores the test's own window. let _ = unsafe { ShowWindow(app, SW_MINIMIZE) }; - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("out of the way", || !visible()); let _ = unsafe { ShowWindow(app, SW_RESTORE) }; - outline::show(window.id, (0x5E, 0xC8, 0xFF)); + outline::show(window.id, (0x5E, 0xC8, 0xFF), None); soon("back", visible); outline::hide(); diff --git a/docs/spec.md b/docs/spec.md index 26acf413..3471e603 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -1768,7 +1768,7 @@ No servidor `botloft` (11), como as do navegador. ### 24.9 App - **Painel "Desktop"** ao lado do chat, o quarto lugar do dock do computador do bot (navegador, terminal, arquivos e desktop, 15.1), com a mesma largura: a foto ao vivo da janela que o bot leu ou em que agiu por último, sobre a mesa na cor do bot, o nome do app e o título da janela, e a linha do que ele acabou de fazer, no idioma do dono ("Clicou em "Save"", "Escreveu em "Nome"", "Escolheu "Large"", "Leu a janela"). No cabeçalho, "Ao vivo" com a foto chegando e **Parar**; parado, o selo "Parado" e um aviso com **Deixar continuar**. Sem janela ainda, o painel explica que ela aparece ali quando o bot ler ou usar um app liberado. Embaixo, a dica do atalho Ctrl+Alt+Esc. As linhas das tools `desktop_*` no chat têm o botão de tela que abre o painel, como as do navegador abrem o dele. Sobre a foto, o cursor do bot desliza até o ponto de cada ação e um anel marca o clique, como no navegador (21.8). O painel abre sozinho quando o bot começa a usar o desktop, isto é, no primeiro `desktop.changed` com janela depois de 2 minutos sem nenhum, sobre o painel que estiver aberto, menos com o navegador nas mãos do dono, e só se o dono não desligou "seguir o bot" (15.1); fechado no meio, as ações seguintes da mesma vez não o abrem de novo. -- **Contorno na tela:** enquanto um bot lê ou age numa janela, ela ganha um contorno na cor do bot, na tela do dono (3 px a 96 DPI, em volta da moldura do DWM). É uma janela própria do daemon, num thread com fila própria: em camadas com uma cor que vira transparente, que deixa os cliques passarem, não pega o foco e não entra na lista de janelas (`WS_EX_TOOLWINDOW`). Não fica sobre tudo: vai logo acima da janela do app na ordem das janelas (atrás da que estava acima dela), então uma janela do dono na frente do app cobre o contorno também. Ela segue a janela a cada 250 ms (posição e ordem) e some com a janela minimizada, escondida ou fechada. Aparece no `desktop.changed` de uma leitura, foto ou ação, e sai 3 s depois de o turno do bot acabar, com um minuto sem uso do desktop no mesmo turno, ou quando o dono o para. Um contorno por vez: o da janela usada por último. Quem decide é uma tarefa do daemon que escuta `desktop.changed` e `bot.state` (`desktop/outline.rs`); a janela fica em `platform/desktop/windows/outline.rs`. +- **Contorno na tela:** enquanto um bot lê ou age numa janela, ela ganha um contorno na cor do bot, na tela do dono (3 px a 96 DPI, em volta da moldura do DWM). É uma janela própria do daemon, num thread com fila própria: em camadas com uma cor que vira transparente, que deixa os cliques passarem, não pega o foco e não entra na lista de janelas (`WS_EX_TOOLWINDOW`). Não fica sobre tudo: vai logo acima da janela do app na ordem das janelas (atrás da que estava acima dela), então uma janela do dono na frente do app cobre o contorno também. Ela segue a janela a cada 250 ms (posição e ordem) e some com a janela minimizada, escondida ou fechada. Aparece no `desktop.changed` de uma leitura, foto ou ação, e sai 3 s depois de o turno do bot acabar, com um minuto sem uso do desktop no mesmo turno, ou quando o dono o para. Um contorno por vez: o da janela usada por último. Dentro dele vai o **cursor do bot** na tela do dono, sem mexer no mouse dele: uma seta na cor do bot, contornada de branco, com o nome do bot numa pílula ao lado (e "•••" enquanto ele escreve). Ele desliza em 350 ms até o ponto de cada ação (`x` e `y` do `DesktopAction`), e um anel na cor do bot cresce onde ele clicou. Depois de só ler, fica onde estava; noutra janela, começa sem cursor até a primeira ação. Desenhado na janela do contorno (GDI, sem suavização, por causa da cor transparente), com um timer de 16 ms só enquanto se move (`platform/desktop/windows/outline_cursor.rs`). Quem decide é uma tarefa do daemon que escuta `desktop.changed` e `bot.state` (`desktop/outline.rs`); a janela fica em `platform/desktop/windows/outline.rs`. - **O que o daemon guarda** (`DesktopState`, em memória, por bot): a janela (`id`, título, app), a última ação (`DesktopAction`: clicar, escrever, escolher ou rolar, o nome do controle, a opção escolhida e o ponto, `x` e `y` em frações da moldura da janela a partir do canto de cima à esquerda: o centro do controle pela acessibilidade, ou o ponto da foto num clique de `desktop_click_at`, sem ponto nas teclas ou com o controle fora da janela; nada depois de só ler), que o app diz no idioma do dono, quando foi, e se o dono o parou. Cada leitura, foto ou ação o atualiza e sai a notificação `desktop.changed` para todos os apps. - **Foto ao vivo:** `desktop.watch {botId}` devolve o estado e a foto mais nova (`DesktopView`) e, enquanto alguma conexão assiste aquele bot, o daemon fotografa a janela dele quatro vezes por segundo (`PrintWindow`, como `desktop_screenshot`) e manda `desktop.frame` só quando a foto mudou, só para quem assiste, como os quadros do navegador (21.7): um app lento recebe a mais nova. Quando a última conexão para (`desktop.unwatch`, outro `desktop.watch` ou a conexão caindo), as fotos param. - **Parar:** `desktop.stop {botId}` marca o bot como parado e `desktop.resume {botId}` o deixa continuar; os dois devolvem o `DesktopState`. Parado, toda tool `desktop_*` dele recusa, de novo depois de esperar o dono ou a vez no desktop, com o aviso de que o dono o parou e que ele deve contar no chat o que fazia. Uma ação já em andamento termina (ela dura no máximo 5 s, 24.5). Parado fica até o dono deixar continuar. Isso vive na memória: um daemon que reinicia começa com ninguém parado.