diff --git a/app/src/features/browser/BrowserPanel.tsx b/app/src/features/browser/BrowserPanel.tsx
index 0d21a5c8..ef041089 100644
--- a/app/src/features/browser/BrowserPanel.tsx
+++ b/app/src/features/browser/BrowserPanel.tsx
@@ -1,7 +1,8 @@
// The bot's own browser beside its chat (spec 21.8): its tabs, the address
// and the page it is on, live, with its cursor and a line about what it
// just did. Watching starts when the panel opens and stops when it closes.
-// The owner can take it into their own hands and give it back (spec 21.10).
+// The owner can take it into their own hands and give it back (spec 21.10),
+// or open it in a window of its own to sign in (spec 21.11).
import { Globe, LoaderCircle, Maximize2, Minimize2, Moon, X } from "lucide-react";
import { useEffect, useRef, useState } from "react";
@@ -23,6 +24,7 @@ import { TabStrip } from "./TabStrip";
import { useBrowserView } from "./useBrowserView";
import { useHands } from "./useHands";
import { useFitPage, useRoom } from "./useRoom";
+import { WindowBar } from "./WindowBar";
/** Around the page in the panel, in px. */
const PAD = 12;
@@ -51,6 +53,7 @@ export function BrowserPanel({
const hands = useHands(bot, state);
const [focused, setFocused] = useState(false);
const ask = state?.ask ?? null;
+ const windowed = state?.window === true;
// The page takes the shape of the room the panel has for it (spec 21.3).
// Notices that come and go are not counted: they never resize the page.
const [inside, body] = useRoom();
@@ -118,6 +121,7 @@ export function BrowserPanel({
className="flex min-h-0 flex-1 flex-col overflow-y-auto"
style={{ padding: PAD }}
>
+ {windowed && }
{status === "failed" ? (
{t.failedBody(SYSTEM)}
@@ -131,7 +135,7 @@ export function BrowserPanel({
)}
) : status === "closed" && !frame ? (
-
+ !windowed &&
) : (
<>
{live && ask && !hands.held && }
@@ -167,8 +171,8 @@ export function BrowserPanel({
{(status !== "open" || !frame) && (
)}
diff --git a/app/src/features/browser/HandsBars.tsx b/app/src/features/browser/HandsBars.tsx
index 0d1071b4..768342f2 100644
--- a/app/src/features/browser/HandsBars.tsx
+++ b/app/src/features/browser/HandsBars.tsx
@@ -1,6 +1,7 @@
// The owner's controls in the browser panel (spec 21.10): taking the
// browser, the bot's request for a hand, and the bar that says the owner
-// has it, with the button that gives it back.
+// has it, with the button that gives it back. Each offers a window of its
+// own too, for sites that refuse to sign in here (spec 21.11).
import { Check, Hand, Keyboard, MousePointerClick } from "lucide-react";
import { useT } from "../../i18n";
@@ -8,6 +9,7 @@ import type { Bot } from "../../lib/protocol.gen";
import { Button } from "../../ui/Button";
import { Callout } from "../../ui/Callout";
import type { Hands } from "./useHands";
+import { WindowButton } from "./WindowBar";
/** The bot asked for a hand and the owner has not taken the browser yet. */
export function AskCallout({ bot, task, hands }: { bot: Bot; task: string; hands: Hands }) {
@@ -16,16 +18,18 @@ export function AskCallout({ bot, task, hands }: { bot: Bot; task: string; hands
{task}
-
+
+
+
+
);
@@ -73,6 +77,7 @@ export function HeldBar({
>
{t.hands.giveBack(bot.name)}
+
{focused ? t.hands.typing : t.hands.clickToType}
@@ -84,9 +89,9 @@ export function HeldBar({
/** Under the live screen: the owner may take the browser at any time. */
export function TakeBar({ bot, hands }: { bot: Bot; hands: Hands }) {
- const t = useT().browser.hands;
+ const t = useT().browser;
return (
-
);
}
diff --git a/app/src/features/browser/WindowBar.tsx b/app/src/features/browser/WindowBar.tsx
new file mode 100644
index 00000000..9ba726f0
--- /dev/null
+++ b/app/src/features/browser/WindowBar.tsx
@@ -0,0 +1,44 @@
+// The bot's browser in a window of its own (spec 21.11): the button that
+// opens it, for sites that refuse to sign in to a browser a program drives,
+// and the bar that says it is open while the bot waits.
+
+import { AppWindow } from "lucide-react";
+import { useT } from "../../i18n";
+import type { Bot } from "../../lib/protocol.gen";
+import { Button } from "../../ui/Button";
+import type { Hands } from "./useHands";
+
+/** Opens the bot's browser in a window of its own. */
+export function WindowButton({ hands }: { hands: Hands }) {
+ const t = useT().browser.window;
+ return (
+
+ );
+}
+
+/** The browser is open in a window of its own. */
+export function WindowBar({ bot }: { bot: Bot }) {
+ const t = useT().browser.window;
+ return (
+
+
+
+
+
{t.title}
+
{t.body(bot.name)}
+
+
+
+ );
+}
diff --git a/app/src/features/browser/useHands.ts b/app/src/features/browser/useHands.ts
index 28150f12..295c52ed 100644
--- a/app/src/features/browser/useHands.ts
+++ b/app/src/features/browser/useHands.ts
@@ -1,5 +1,6 @@
// The owner's hands in a bot's browser (spec 21.10): taking it, giving it
-// back, sending what they do on the page, and moving between its tabs.
+// back, sending what they do on the page, moving between its tabs, and
+// opening it in a window of its own (spec 21.11).
// Events go out in order and without waiting for each other: the daemon
// queues them for the page.
@@ -15,6 +16,8 @@ export interface Hands {
busy: boolean;
take(): Promise;
release(): Promise;
+ /** Opens the browser in a window of its own, to sign in. */
+ window(): Promise;
send(input: BrowserInput): void;
/** Opens a blank tab; `true` once the daemon took the request. */
newTab(): Promise;
@@ -46,6 +49,14 @@ export function useHands(bot: Pick, state: BrowserState | null): Hand
setBusy(false);
}, [api, botId, putBrowser, t.hands.giveBackFailed]);
+ const openWindow = useCallback(async () => {
+ setBusy(true);
+ await attempt(t.window.openFailed, async () => {
+ putBrowser(await api.call("browser.window", { botId }));
+ });
+ setBusy(false);
+ }, [api, botId, putBrowser, t.window.openFailed]);
+
const send = useCallback(
(input: BrowserInput) => {
// One lost event is not worth a message; the page shows what arrived.
@@ -73,6 +84,7 @@ export function useHands(bot: Pick, state: BrowserState | null): Hand
busy,
take,
release,
+ window: openWindow,
send,
newTab,
switchTab,
diff --git a/app/src/features/browser/window.test.tsx b/app/src/features/browser/window.test.tsx
new file mode 100644
index 00000000..c63955e6
--- /dev/null
+++ b/app/src/features/browser/window.test.tsx
@@ -0,0 +1,64 @@
+import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import { afterEach, describe, expect, test } from "vitest";
+import { FakeBotloft } from "../../lib/fake";
+import { crewOpened, openBot, renderApp } from "../../test/app";
+
+afterEach(cleanup);
+
+/** A crew "Ops" with @scout, its browser open on a sign-in page, watched. */
+async function openScout() {
+ const fake = new FakeBotloft();
+ const ops = fake.addCrew("Ops");
+ const scout = fake.addBot(ops.id, "Scout", "Finds sources");
+ fake.setBotState(scout.id, "busy");
+ fake.browser.open(scout.id, "https://accounts.google.com/", "Sign in");
+ renderApp(fake);
+ await crewOpened("Ops");
+ openBot("Scout");
+ await screen.findByRole("list", { name: "Messages" });
+ fireEvent.click(screen.getByRole("button", { name: /^Show browser/ }));
+ await waitFor(() => expect(fake.browser.watching).toBe(scout.id));
+ act(() => {
+ fake.browser.frame(scout.id, "AAAA");
+ });
+ return { fake, scout };
+}
+
+const panel = () => screen.getByRole("complementary", { name: "Scout's browser" });
+
+describe("the browser in a window of its own", () => {
+ test("opens for signing in, says the bot waits, and goes back when the window closes", async () => {
+ const { fake, scout } = await openScout();
+ expect(within(panel()).getByText(/like Google/)).toBeDefined();
+ fireEvent.click(within(panel()).getByRole("button", { name: "Sign in in a window" }));
+
+ expect(await within(panel()).findAllByText("Open in a window")).not.toHaveLength(0);
+ expect(within(panel()).getByText(/Scout waits meanwhile and keeps the login/)).toBeDefined();
+ expect(
+ fake.calls.filter((call) => call.method === "browser.window").map((call) => call.params),
+ ).toEqual([{ botId: scout.id }]);
+ expect(within(panel()).queryByRole("button", { name: "Take control" })).toBeNull();
+
+ act(() => {
+ fake.browser.closeWindow(scout.id);
+ });
+ await waitFor(() => expect(within(panel()).queryByText(/keeps the login/)).toBeNull());
+ });
+
+ test("is offered when the bot asks for a hand", async () => {
+ const { fake, scout } = await openScout();
+ act(() => {
+ fake.browser.ask(scout.id, "Sign in to your Google account");
+ });
+ const asking = await within(panel()).findByText("Sign in to your Google account");
+ expect(asking).toBeDefined();
+ fireEvent.click(within(panel()).getByRole("button", { name: "Sign in in a window" }));
+ expect(await within(panel()).findByText(/keeps the login/)).toBeDefined();
+
+ // Closing the window is the owner being done.
+ act(() => {
+ fake.browser.closeWindow(scout.id);
+ });
+ expect(await screen.findByText("You did it: Sign in to your Google account")).toBeDefined();
+ });
+});
diff --git a/app/src/i18n/en/browser.ts b/app/src/i18n/en/browser.ts
index d28e93f0..33688120 100644
--- a/app/src/i18n/en/browser.ts
+++ b/app/src/i18n/en/browser.ts
@@ -79,6 +79,15 @@ export const browser = {
typing: "What you type goes to the page",
screen: (bot: string) => `${bot}'s browser, in your hands`,
},
+ window: {
+ open: "Sign in in a window",
+ why: (bot: string) =>
+ `For sites that refuse to sign in here, like Google: ${bot}'s browser opens in a window of its own, and the login stays for ${bot}.`,
+ openFailed: "Could not open the window",
+ title: "Open in a window",
+ body: (bot: string) =>
+ `Sign in in the window that opened, then close it. ${bot} waits meanwhile and keeps the login.`,
+ },
help: {
needs: (bot: string) => `${bot} needs you in the browser`,
asks: (bot: string, task: string) => `${bot} asks: ${task}`,
diff --git a/app/src/i18n/es/browser.ts b/app/src/i18n/es/browser.ts
index 9467b04b..cca1a261 100644
--- a/app/src/i18n/es/browser.ts
+++ b/app/src/i18n/es/browser.ts
@@ -79,6 +79,15 @@ export const browser: Messages["browser"] = {
typing: "Lo que escribes va a la página",
screen: (bot) => `Navegador de ${bot}, en tus manos`,
},
+ window: {
+ open: "Iniciar sesión en una ventana",
+ why: (bot) =>
+ `Para sitios que rechazan el inicio de sesión aquí, como Google: el navegador de ${bot} se abre en una ventana propia y la sesión queda para ${bot}.`,
+ openFailed: "No se pudo abrir la ventana",
+ title: "Abierto en una ventana",
+ body: (bot) =>
+ `Inicia sesión en la ventana que se abrió y luego ciérrala. ${bot} espera mientras tanto y conserva la sesión.`,
+ },
help: {
needs: (bot) => `${bot} te necesita en el navegador`,
asks: (bot, task) => `${bot} pide: ${task}`,
diff --git a/app/src/i18n/pt-BR/browser.ts b/app/src/i18n/pt-BR/browser.ts
index 1c8a23cd..fe7ef25c 100644
--- a/app/src/i18n/pt-BR/browser.ts
+++ b/app/src/i18n/pt-BR/browser.ts
@@ -81,6 +81,15 @@ export const browser: Messages["browser"] = {
typing: "O que você digita vai para a página",
screen: (bot) => `Navegador de ${bot}, nas suas mãos`,
},
+ window: {
+ open: "Entrar numa janela",
+ why: (bot) =>
+ `Para sites que recusam o login aqui, como o Google: o navegador de ${bot} abre numa janela própria, e o login fica com ${bot}.`,
+ openFailed: "Não foi possível abrir a janela",
+ title: "Aberto numa janela",
+ body: (bot) =>
+ `Entre na conta na janela que abriu e depois feche-a. ${bot} espera enquanto isso e fica com o login.`,
+ },
help: {
needs: (bot) => `${bot} precisa de você no navegador`,
asks: (bot, task) => `${bot} pede: ${task}`,
diff --git a/app/src/lib/fakeBrowser.ts b/app/src/lib/fakeBrowser.ts
index 6e23e376..6b525daa 100644
--- a/app/src/lib/fakeBrowser.ts
+++ b/app/src/lib/fakeBrowser.ts
@@ -1,7 +1,7 @@
// The fake daemon's browsers (spec 21.7): each bot's state and tabs, the
// live frames of the one being watched and what the bot does, for the
// browser panel, the size the panel gives the page (spec 21.3), and the
-// owner's hands in it (spec 21.10).
+// owner's hands in it (spec 21.10) or in a window of its own (spec 21.11).
import type { FakeBotloft, Handlers } from "./fake";
import { fitting, PAGE, type PageSize, typed } from "./fakePages";
@@ -52,6 +52,7 @@ export class FakeBrowser {
control: "bot",
resting: false,
ask: null,
+ window: false,
updatedAt: this.fake.now,
}
);
@@ -125,6 +126,15 @@ export class FakeBrowser {
this.set(botId, { ask: null, control: "bot" });
}
+ /** The owner closed the window of the bot's browser: it is the bot's again. */
+ closeWindow(botId: BotId): void {
+ const approvalId = this.asks.get(botId);
+ if (approvalId) {
+ void this.fake.call("approvals.answer", { approvalId, allow: true });
+ }
+ this.set(botId, { window: false, control: "bot" });
+ }
+
/** The size of the bot's page now. */
size(botId: BotId): PageSize {
return this.sizes.get(botId) ?? PAGE;
@@ -174,7 +184,8 @@ export class FakeBrowser {
/** The watched browser leaves the owner's hands, if it was in them. */
private release(): BrowserState | null {
const botId = this.watching;
- if (botId === null || this.state(botId).control !== "owner") {
+ const state = botId === null ? null : this.state(botId);
+ if (botId === null || state?.control !== "owner" || state.window) {
return null;
}
return this.set(botId, { control: "bot" });
@@ -210,9 +221,11 @@ export class FakeBrowser {
| "browser.newTab"
| "browser.switchTab"
| "browser.open"
+ | "browser.window"
> {
return {
- "browser.list": () => [...this.states.values()].filter((state) => state.status !== "closed"),
+ "browser.list": () =>
+ [...this.states.values()].filter((state) => state.status !== "closed" || state.window),
"browser.watch": ({ botId }) => {
this.fake.bot(botId, false);
this.unwatch();
@@ -296,6 +309,17 @@ export class FakeBrowser {
this.repaint(botId);
return null;
},
+ "browser.window": ({ botId }) => {
+ if (this.watching !== botId) {
+ throw conflict("watch this browser before opening it in a window");
+ }
+ if (this.state(botId).window) {
+ throw conflict("the browser is already open in a window");
+ }
+ this.frames.delete(botId);
+ const away = { status: "closed", control: "owner", resting: false, window: true } as const;
+ return this.setTabs(botId, [], away);
+ },
};
}
}
diff --git a/app/src/lib/protocol.gen.ts b/app/src/lib/protocol.gen.ts
index 0efc3d80..ed9f4261 100644
--- a/app/src/lib/protocol.gen.ts
+++ b/app/src/lib/protocol.gen.ts
@@ -1212,6 +1212,12 @@ resting: boolean,
* What the bot asked the owner to do in it, while it waits.
*/
ask: string | null,
+/**
+ * The owner has it open in a window of its own, to sign in where a
+ * site refuses a browser a program drives (spec 21.11). It is
+ * `closed` meanwhile, and the bot's tools wait.
+ */
+window: boolean,
/**
* Unix time in milliseconds.
*/
@@ -1282,8 +1288,8 @@ buttons: number,
clicks: number, modifiers: number, } | { "kind": "wheel", x: number, y: number, dx: number, dy: number, modifiers: number, } | { "kind": "key", key: string, code: string, modifiers: number, } | { "kind": "text", text: string, };
/**
- * `browser.take`, `browser.release`, `browser.reload` and
- * `browser.newTab`.
+ * `browser.take`, `browser.release`, `browser.reload`, `browser.newTab`
+ * and `browser.window`.
*/
export type BrowserControlParams = { botId: BotId, };
@@ -1417,6 +1423,7 @@ export interface RpcMethods {
"browser.resize": { params: BrowserResizeParams; result: null };
"browser.take": { params: BrowserControlParams; result: BrowserState };
"browser.release": { params: BrowserControlParams; result: BrowserState };
+ "browser.window": { params: BrowserControlParams; result: BrowserState };
"browser.input": { params: BrowserInputParams; result: null };
"browser.reload": { params: BrowserControlParams; result: null };
"browser.newTab": { params: BrowserControlParams; result: null };
diff --git a/app/src/store/app.ts b/app/src/store/app.ts
index 488ff366..d9611f26 100644
--- a/app/src/store/app.ts
+++ b/app/src/store/app.ts
@@ -342,7 +342,9 @@ export function applyEvent(state: AppState, event: ServerEvent): Partial {
const { [browser.botId]: _, ...rest } = state.browsers;
return {
- browsers: browser.status === "closed" ? rest : { ...rest, [browser.botId]: browser },
+ // Closed is no browser, unless it is open in a window of its own.
+ browsers:
+ browser.status === "closed" && !browser.window ? rest : { ...rest, [browser.botId]: browser },
};
}
diff --git a/crates/botloft-core/src/protocol/browser.rs b/crates/botloft-core/src/protocol/browser.rs
index 90be0617..48a6aff3 100644
--- a/crates/botloft-core/src/protocol/browser.rs
+++ b/crates/botloft-core/src/protocol/browser.rs
@@ -55,6 +55,10 @@ pub struct BrowserState {
pub resting: bool,
/// What the bot asked the owner to do in it, while it waits.
pub ask: Option,
+ /// The owner has it open in a window of its own, to sign in where a
+ /// site refuses a browser a program drives (spec 21.11). It is
+ /// `closed` meanwhile, and the bot's tools wait.
+ pub window: bool,
/// Unix time in milliseconds.
pub updated_at: i64,
}
@@ -72,6 +76,7 @@ impl BrowserState {
control: BrowserControl::Bot,
resting: false,
ask: None,
+ window: false,
updated_at: now,
}
}
diff --git a/crates/botloft-core/src/protocol/browser_input.rs b/crates/botloft-core/src/protocol/browser_input.rs
index 0677aaf2..72a2a2a2 100644
--- a/crates/botloft-core/src/protocol/browser_input.rs
+++ b/crates/botloft-core/src/protocol/browser_input.rs
@@ -91,8 +91,8 @@ impl std::fmt::Debug for BrowserInput {
}
}
-/// `browser.take`, `browser.release`, `browser.reload` and
-/// `browser.newTab`.
+/// `browser.take`, `browser.release`, `browser.reload`, `browser.newTab`
+/// and `browser.window`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
#[cfg_attr(test, derive(ts_rs::TS))]
diff --git a/crates/botloft-core/src/protocol/export_browser.rs b/crates/botloft-core/src/protocol/export_browser.rs
index e0dfcb00..d6c4fa92 100644
--- a/crates/botloft-core/src/protocol/export_browser.rs
+++ b/crates/botloft-core/src/protocol/export_browser.rs
@@ -50,6 +50,7 @@ pub(super) fn methods(out: &mut Out) {
let state = out.name::();
out.method(method::BROWSER_TAKE, &control, &state);
out.method(method::BROWSER_RELEASE, &control, &state);
+ out.method(method::BROWSER_WINDOW, &control, &state);
out.method(
method::BROWSER_INPUT,
&out.name::(),
diff --git a/crates/botloft-core/src/protocol/mod.rs b/crates/botloft-core/src/protocol/mod.rs
index f1704b97..f07a28bc 100644
--- a/crates/botloft-core/src/protocol/mod.rs
+++ b/crates/botloft-core/src/protocol/mod.rs
@@ -143,6 +143,7 @@ pub mod method {
pub const BROWSER_NEW_TAB: &str = "browser.newTab";
pub const BROWSER_SWITCH_TAB: &str = "browser.switchTab";
pub const BROWSER_OPEN: &str = "browser.open";
+ pub const BROWSER_WINDOW: &str = "browser.window";
pub const SCREENS_LIST: &str = "screens.list";
pub const QUESTIONS_LIST: &str = "questions.list";
pub const QUESTIONS_ANSWER: &str = "questions.answer";
diff --git a/crates/botloftd/src/browser/agent.rs b/crates/botloftd/src/browser/agent.rs
new file mode 100644
index 00000000..b78e5801
--- /dev/null
+++ b/crates/botloftd/src/browser/agent.rs
@@ -0,0 +1,110 @@
+//! What the browser tells sites it is (spec 21.3): the user agent of the
+//! Edge it runs without the word `Headless`, with the brands and platform a
+//! site reads from `navigator.userAgentData`. Overriding the user agent
+//! alone would leave those empty, which no common browser does.
+
+use std::time::Duration;
+
+use serde_json::{Value, json};
+use tokio::sync::mpsc;
+use tracing::debug;
+
+use super::BrowserError;
+use super::cdp::{Cdp, CdpEvent};
+
+/// A page of the browser's own that has `navigator.userAgentData`, which
+/// only secure pages have (`about:blank` does not).
+const SECURE_PAGE: &str = "chrome://version";
+/// What `navigator.userAgentData` says, once the page has it.
+const READ_METADATA: &str = "navigator.userAgentData ? navigator.userAgentData\
+ .getHighEntropyValues(['architecture', 'bitness', 'fullVersionList', 'model', \
+ 'platformVersion', 'wow64']) : null";
+/// Tries while the page loads, 50 ms apart.
+const TRIES: usize = 40;
+
+/// The params of `Emulation.setUserAgentOverride` for every tab. Run before
+/// anything else reads `events`: the page it reads from opens and closes
+/// here, and its events are dropped.
+pub async fn read(
+ cdp: &Cdp,
+ events: &mut mpsc::UnboundedReceiver,
+) -> Result {
+ let version = cdp.call(None, "Browser.getVersion", json!({})).await?;
+ let user_agent = version["userAgent"]
+ .as_str()
+ .unwrap_or_default()
+ .replace("HeadlessChrome", "Chrome");
+ let mut params = json!({ "userAgent": user_agent });
+ match metadata(cdp).await {
+ Ok(Some(metadata)) => params["userAgentMetadata"] = metadata,
+ Ok(None) => debug!("browser: the user agent data did not come"),
+ Err(err) => debug!("browser: reading the user agent data: {err}"),
+ }
+ while events.try_recv().is_ok() {}
+ Ok(params)
+}
+
+/// The brands and platform a secure page reads, in the shape
+/// `Emulation.UserAgentMetadata` takes.
+async fn metadata(cdp: &Cdp) -> Result