Skip to content

Commit 178ae3e

Browse files
committed
fix(browser): clicks land on a scaled screen, and the bot's browser stops waiting for nothing
Measured with the real Edge. On a screen scaled above 100% the page was drawn sharper (deviceScaleFactor) with a Page.captureScreenshot per frame, and the browser then read the bot's and the owner's clicks divided by the factor: 4 of 12 size and scale combinations landed. The page is now always drawn in its own pixels; the panel scales the frame. 16 of 16 land. Also: after an action the daemon waited up to 3 s for a network that never goes quiet on pages with analytics, ads or chat (3.2 s a click); a page that was already busy before the action now waits 0.6 s. While the owner watches, the tool waited 560 ms for the cursor and aimed before the wait, so a page that changed meanwhile took the click at the wrong point: the wait is 300 ms (cursor glide 260 ms) and the element is found again after it. A button under a fixed header was clicked on the header: other scroll alignments and points of the element are tried, and when none is free the page's own click is used and the bot is told. Tests: scaled-screen clicks (bot and owner, 100 to 200%), a never-quiet page, a covered button. Spec 21.3, 21.4, 21.7 and 19 updated.
1 parent ca38299 commit 178ae3e

15 files changed

Lines changed: 378 additions & 141 deletions

File tree

‎app/src/lib/protocol.gen.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1745,8 +1745,9 @@ export type BrowserWatchParams = { botId: BotId, };
17451745
export type BrowserResizeParams = { botId: BotId, width: number, height: number,
17461746
/**
17471747
* The screen's pixels per pixel of the app, in percent (its
1748-
* `devicePixelRatio` times 100): the page is drawn that sharp for the
1749-
* owner, up to twice (spec 21.3). Absent, 100.
1748+
* `devicePixelRatio` times 100). The daemon ignores it: the page is
1749+
* always drawn in its own pixels, because a sharper page made clicks
1750+
* land at the wrong point (spec 21.3, 19). Absent, 100.
17501751
*/
17511752
scale?: number, };
17521753

‎app/src/motion.css‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -174,8 +174,8 @@
174174
pointer-events: none;
175175
translate: -4px -3px;
176176
transition:
177-
left 520ms var(--ease-out),
178-
top 520ms var(--ease-out);
177+
left 260ms var(--ease-out),
178+
top 260ms var(--ease-out);
179179
}
180180

181181
.bot-cursor > span {
@@ -190,7 +190,7 @@
190190
border: 3px solid;
191191
border-radius: 9999px;
192192
pointer-events: none;
193-
animation: ripple 700ms var(--ease-out) 520ms both;
193+
animation: ripple 700ms var(--ease-out) 260ms both;
194194
}
195195

196196
.bot-typing {

‎crates/botloft-core/src/protocol/browser.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -167,8 +167,9 @@ pub struct BrowserResizeParams {
167167
pub width: u32,
168168
pub height: u32,
169169
/// The screen's pixels per pixel of the app, in percent (its
170-
/// `devicePixelRatio` times 100): the page is drawn that sharp for the
171-
/// owner, up to twice (spec 21.3). Absent, 100.
170+
/// `devicePixelRatio` times 100). The daemon ignores it: the page is
171+
/// always drawn in its own pixels, because a sharper page made clicks
172+
/// land at the wrong point (spec 21.3, 19). Absent, 100.
172173
#[serde(default, skip_serializing_if = "Option::is_none")]
173174
#[cfg_attr(test, ts(optional))]
174175
pub scale: Option<u32>,

‎crates/botloftd/src/browser/actions.js‎

Lines changed: 53 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,30 +19,73 @@
1919
return { left, top, width: rect.width, height: rect.height };
2020
};
2121

22-
// Where to click an element, after scrolling it into the middle of the
23-
// screen, and what else is on top of that point, if anything.
22+
// Whether something else is on top of a point of an element: true when
23+
// the point shows the element, a part of it or what it sits in.
24+
const shows = (el, x, y) => {
25+
const hit = el.ownerDocument.elementFromPoint(x, y);
26+
return !!hit && (hit === el || el.contains(hit) || hit.contains(el));
27+
};
28+
29+
// A point inside the element that is not covered, the middle first, then
30+
// the corners and edges of its middle part, or null. Returns the fraction
31+
// of the box it is at.
32+
const freePoint = (el, local) => {
33+
const view = el.ownerDocument.defaultView;
34+
for (const fy of [0.5, 0.2, 0.8]) {
35+
for (const fx of [0.5, 0.2, 0.8]) {
36+
const x = local.left + local.width * fx;
37+
const y = local.top + local.height * fy;
38+
if (x < 0 || y < 0 || x >= view.innerWidth || y >= view.innerHeight) continue;
39+
if (shows(el, x, y)) return { fx, fy };
40+
}
41+
}
42+
return null;
43+
};
44+
45+
// Where to click an element. It scrolls into the middle of the screen;
46+
// when a fixed header, a banner or a chat widget covers it there, it tries
47+
// the other alignments and the other points of the element, and when
48+
// nothing is free it says so (`script`), so the click can go through the
49+
// page instead of the mouse. Also what is still on top, if anything.
2450
const point = (ref) => {
2551
const el = get(ref);
2652
if (!el) return { missing: true, next: next() };
27-
el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
53+
let pick = null;
54+
for (const block of ["center", "end", "start", "nearest"]) {
55+
el.scrollIntoView({ block, inline: "center", behavior: "instant" });
56+
const local = el.getBoundingClientRect();
57+
if (local.width === 0 || local.height === 0) break;
58+
pick = freePoint(el, local);
59+
if (pick) break;
60+
}
61+
if (!pick) el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
2862
const box = topRect(el);
2963
const role = roleOf(el, styleOf(el), "auto") ?? el.localName;
3064
const label = clean(nameOf(el), 60);
3165
if (box.width === 0 || box.height === 0) return { hidden: true, role, label, next: next() };
32-
const x = Math.min(Math.max(box.left + box.width / 2, 0), innerWidth - 1);
33-
const y = Math.min(Math.max(box.top + box.height / 2, 0), innerHeight - 1);
66+
const [fx, fy] = pick ? [pick.fx, pick.fy] : [0.5, 0.5];
67+
const x = Math.min(Math.max(box.left + box.width * fx, 0), innerWidth - 1);
68+
const y = Math.min(Math.max(box.top + box.height * fy, 0), innerHeight - 1);
3469
const local = el.getBoundingClientRect();
3570
const hit = el.ownerDocument.elementFromPoint(
36-
local.left + local.width / 2,
37-
local.top + local.height / 2,
71+
local.left + local.width * fx,
72+
local.top + local.height * fy,
3873
);
3974
let cover = null;
40-
if (hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
75+
if (!pick && hit && hit !== el && !el.contains(hit) && !hit.contains(el)) {
4176
let shown = hit;
4277
while (shown && !roleOf(shown, styleOf(shown), "auto")) shown = shown.parentElement;
4378
cover = shown ? token(shown, roleOf(shown, styleOf(shown), "auto")) : `<${hit.localName}>`;
4479
}
45-
return { x, y, role, label, text: isTextField(el), cover, next: next() };
80+
return { x, y, role, label, text: isTextField(el), cover, script: !pick && cover !== null, next: next() };
81+
};
82+
83+
// The page's own click on an element, for when the mouse cannot reach it.
84+
const clickScript = (ref) => {
85+
const el = get(ref);
86+
if (!el) return { missing: true };
87+
el.click();
88+
return { ok: true };
4689
};
4790

4891
// Focuses a text field and selects what it holds, so typing replaces it.
@@ -112,5 +155,5 @@
112155
return el && el !== document.body && isTextField(el) ? meaning(el) : { none: true };
113156
};
114157

115-
return { point, prepareType, select, scrollEnd, at, focused };
158+
return { point, clickScript, prepareType, select, scrollEnd, at, focused };
116159
}

‎crates/botloftd/src/browser/events.rs‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -211,14 +211,7 @@ impl Session {
211211
size("deviceWidth", viewport.width),
212212
size("deviceHeight", viewport.height),
213213
);
214-
// The screencast draws in the page's own pixels, however sharp the
215-
// page is (spec 19): on a sharper screen its frame only says the page
216-
// changed, and the frame sent is a screenshot, which is that sharp.
217-
let sharp = active && viewport.scale > 100;
218-
if active
219-
&& !sharp
220-
&& let Some(data) = params["data"].as_str()
221-
{
214+
if active && let Some(data) = params["data"].as_str() {
222215
hooks.frames.send_replace(Some(Arc::new(BrowserFrame {
223216
bot_id: hooks.bot.clone(),
224217
data: data.to_owned(),
@@ -229,26 +222,7 @@ impl Session {
229222
let cdp = self.cdp.clone();
230223
let session = session.to_owned();
231224
let ack = json!({ "sessionId": params["sessionId"] });
232-
let (frames, bot) = (hooks.frames.clone(), hooks.bot.clone());
233225
tokio::spawn(async move {
234-
if sharp {
235-
let shot = json!({ "format": "jpeg", "quality": 60, "optimizeForSpeed": true });
236-
let taken = cdp
237-
.call(Some(&session), "Page.captureScreenshot", shot)
238-
.await;
239-
if let Some(data) = taken
240-
.ok()
241-
.and_then(|shot| shot["data"].as_str().map(str::to_owned))
242-
{
243-
frames.send_replace(Some(Arc::new(BrowserFrame {
244-
bot_id: bot,
245-
data,
246-
width,
247-
height,
248-
})));
249-
}
250-
}
251-
// Confirmed only after, so screenshots never pile up.
252226
tokio::time::sleep(gap).await;
253227
cdp.send(Some(&session), "Page.screencastFrameAck", ack);
254228
});

‎crates/botloftd/src/browser/page.rs‎

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ pub struct Done {
1919
pub label: Option<String>,
2020
/// Another element was on top of the one clicked.
2121
pub cover: Option<String>,
22+
/// Nothing of the element was free of what covered it, so the click went
23+
/// through the page's own script and not the mouse.
24+
pub scripted: bool,
2225
/// What `browser_select` chose.
2326
pub chosen: Option<String>,
2427
}
@@ -28,6 +31,8 @@ pub struct Done {
2831
pub struct Aim {
2932
reference: String,
3033
point: Value,
34+
/// Covered whatever the point: only the page's own script can click it.
35+
script: bool,
3136
pub done: Done,
3237
}
3338

@@ -81,10 +86,12 @@ impl Session {
8186
point: point["x"].as_f64().zip(point["y"].as_f64()),
8287
label,
8388
cover: point["cover"].as_str().map(str::to_owned),
89+
scripted: false,
8490
chosen: None,
8591
};
8692
Ok(Aim {
8793
reference: reference.to_owned(),
94+
script: point["script"] == true,
8895
point,
8996
done,
9097
})
@@ -114,8 +121,27 @@ impl Session {
114121
Ok(())
115122
}
116123

117-
pub async fn click(&self, Aim { done, .. }: Aim) -> Result<Done, BrowserError> {
124+
pub async fn click(
125+
&self,
126+
Aim {
127+
reference,
128+
script,
129+
mut done,
130+
..
131+
}: Aim,
132+
) -> Result<Done, BrowserError> {
118133
let marks = self.marks();
134+
if script {
135+
let result = self
136+
.reader(&format!("clickScript({})", json!(reference)))
137+
.await?;
138+
if result["missing"] == true {
139+
return Err(BrowserError::Stale(reference));
140+
}
141+
done.scripted = true;
142+
self.settle(marks).await;
143+
return Ok(done);
144+
}
119145
self.mouse_click(done.point.unwrap_or_default()).await?;
120146
self.settle(marks).await;
121147
Ok(done)
@@ -129,10 +155,16 @@ impl Session {
129155
submit: bool,
130156
) -> Result<Done, BrowserError> {
131157
let Aim {
132-
reference, done, ..
158+
reference,
159+
script,
160+
done,
161+
..
133162
} = aim;
134163
let marks = self.marks();
135-
self.mouse_click(done.point.unwrap_or_default()).await?;
164+
// A field nothing frees is still focused by `prepareType` below.
165+
if !script {
166+
self.mouse_click(done.point.unwrap_or_default()).await?;
167+
}
136168
let ready = self
137169
.reader(&format!("prepareType({})", json!(reference)))
138170
.await?;

‎crates/botloftd/src/browser/session.rs‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,6 @@ pub(super) const FRAME_GAP: Duration = Duration::from_millis(66);
2727
/// ...and ~30 while the browser is in the owner's hands, so the page keeps
2828
/// up with them as they scroll and point (spec 21.8).
2929
pub(super) const QUICK_GAP: Duration = Duration::from_millis(33);
30-
/// The largest side of a frame that only says the page changed.
31-
const SIGNAL: u32 = 64;
3230
/// How long the first tab may take to show up.
3331
const FIRST_TAB_TIMEOUT: Duration = Duration::from_secs(5);
3432

@@ -259,16 +257,9 @@ impl Session {
259257

260258
pub(super) async fn cast(&self, session: &str, on: bool) {
261259
let result = if on {
262-
// Frames come the size of the page, however tall it is. On a
263-
// sharper screen they only say the page changed (`events.rs`),
264-
// so they come small.
265-
let (max_width, max_height) = if self.viewport().scale > 100 {
266-
(SIGNAL, SIGNAL)
267-
} else {
268-
(MAX_WIDTH, MAX_HEIGHT)
269-
};
260+
// Frames come the size of the page, however tall it is.
270261
let params = json!({
271-
"format": "jpeg", "quality": 60, "maxWidth": max_width, "maxHeight": max_height,
262+
"format": "jpeg", "quality": 60, "maxWidth": MAX_WIDTH, "maxHeight": MAX_HEIGHT,
272263
});
273264
self.cdp
274265
.call(Some(session), "Page.startScreencast", params)

‎crates/botloftd/src/browser/settle.rs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,18 @@ use super::session::Session;
88
const LOAD_WAIT: Duration = Duration::from_secs(15);
99
const QUIET: Duration = Duration::from_millis(500);
1010
const QUIET_WAIT: Duration = Duration::from_secs(3);
11+
/// The same wait on a page that was never quiet before the action (an
12+
/// analytics script, a chat widget, a live feed): its network will not go
13+
/// quiet because of the click, so waiting for it only costs the bot 3 s per
14+
/// action (measured: 3.2 s a click, against 0.17 s on a quiet page).
15+
const NOISY_WAIT: Duration = Duration::from_millis(600);
1116

1217
/// The active tab before an action, to tell whether it navigated.
1318
pub(super) struct Marks {
1419
target: String,
1520
navigations: u64,
21+
/// The network was busy just before the action, whatever the action does.
22+
noisy: bool,
1623
}
1724

1825
impl Session {
@@ -22,6 +29,8 @@ impl Session {
2229
Marks {
2330
target: tab.map(|tab| tab.target.clone()).unwrap_or_default(),
2431
navigations: tab.map_or(0, |tab| tab.navigations),
32+
noisy: tab
33+
.is_some_and(|tab| !tab.inflight.is_empty() || tab.network_at.elapsed() < QUIET),
2534
}
2635
}
2736

@@ -46,7 +55,7 @@ impl Session {
4655
}
4756
let _ = tokio::time::timeout(left, notified).await;
4857
}
49-
let deadline = Instant::now() + QUIET_WAIT;
58+
let deadline = Instant::now() + if before.noisy { NOISY_WAIT } else { QUIET_WAIT };
5059
loop {
5160
let notified = self.changed.notified();
5261
let quiet_for = {

0 commit comments

Comments
 (0)