Skip to content

Commit 8e2a2e0

Browse files
authored
Merge pull request #35 from httpsphl/feat/browser-hands
feat: the owner takes a bot's browser into their own hands (N3)
2 parents dcc2af6 + 1942061 commit 8e2a2e0

48 files changed

Lines changed: 2447 additions & 134 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎app/src/dev/preview.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { PROTOCOL_VERSION } from "../lib/protocol.gen";
1313
import { seedBrowser } from "./seedBrowser";
1414
import { seedChats } from "./seedChats";
1515
import { seedChief } from "./seedChief";
16+
import { seedHands } from "./seedHands";
1617
import { seedRoutines } from "./seedRoutines";
1718
import { seedScreens } from "./seedScreens";
1819

@@ -115,6 +116,7 @@ function seed(fake: FakeBotloft): void {
115116
seedRoutines(fake, { scout: scout.id, analyst: analyst.id });
116117
seedBrowser(fake, scout.id);
117118
seedScreens(fake, research.id);
119+
seedHands(fake, research.id);
118120
const ops = fake.addCrew("Ops");
119121
const deploy = fake.addBot(ops.id, "Deploy", "Ships the site on Fridays");
120122
const watcher = fake.addBot(ops.id, "Watcher", "Keeps an eye on the error log");

‎app/src/dev/seedHands.ts‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Dev only: a Clerk that asks the owner to sign in for it, in the fake
2+
// preview (spec 21.10). The sign-in page is drawn on a canvas and answers
3+
// the owner's clicks and keys, so taking the browser can be tried end to end.
4+
5+
import type { FakeBotloft } from "../lib/fake";
6+
import type { BrowserInput, CrewId } from "../lib/protocol.gen";
7+
8+
const W = 1280;
9+
const H = 800;
10+
const FIELDS = { user: 300, password: 390 } as const;
11+
type Field = keyof typeof FIELDS;
12+
13+
interface Form {
14+
user: string;
15+
password: string;
16+
field: Field | null;
17+
signedIn: boolean;
18+
}
19+
20+
function draw(form: Form): string {
21+
const canvas = document.createElement("canvas");
22+
canvas.width = W;
23+
canvas.height = H;
24+
const g = canvas.getContext("2d");
25+
if (!g) {
26+
return "";
27+
}
28+
g.fillStyle = "#f7f4ee";
29+
g.fillRect(0, 0, W, H);
30+
g.fillStyle = "#3b2a1a";
31+
g.font = "600 36px Georgia, serif";
32+
if (form.signedIn) {
33+
g.fillText(`Welcome back, ${form.user || "baker"}`, 440, 220);
34+
g.font = "20px system-ui, sans-serif";
35+
const orders = ["#1042 · 25 kg flour", "#1043 · 10 kg rye", "#1044 · 5 kg semolina"];
36+
for (const [row, line] of orders.entries()) {
37+
g.fillText(`Order ${line}`, 440, 300 + row * 48);
38+
}
39+
return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? "";
40+
}
41+
g.fillText("Flour Co. · Sign in", 440, 220);
42+
for (const name of ["user", "password"] as const) {
43+
const y = FIELDS[name];
44+
g.fillStyle = "#ffffff";
45+
g.strokeStyle = form.field === name ? "#c2410c" : "#b5a896";
46+
g.lineWidth = form.field === name ? 3 : 2;
47+
g.beginPath();
48+
g.roundRect(440, y, 400, 52, 12);
49+
g.fill();
50+
g.stroke();
51+
const value = name === "password" ? "•".repeat(form.password.length) : form.user;
52+
g.fillStyle = value ? "#1f1409" : "#9a8c7a";
53+
g.font = "20px system-ui, sans-serif";
54+
g.fillText(value || (name === "user" ? "E-mail" : "Password"), 460, y + 33);
55+
}
56+
g.fillStyle = "#c2410c";
57+
g.beginPath();
58+
g.roundRect(440, 480, 400, 52, 26);
59+
g.fill();
60+
g.fillStyle = "#ffffff";
61+
g.font = "600 20px system-ui, sans-serif";
62+
g.fillText("Sign in", 606, 513);
63+
return canvas.toDataURL("image/jpeg", 0.7).split(",")[1] ?? "";
64+
}
65+
66+
/** The page's answer to one of the owner's events; true once they sign in. */
67+
function handle(form: Form, input: BrowserInput): boolean {
68+
if (input.kind === "mouse" && input.action === "down") {
69+
const inside = (top: number) =>
70+
input.x >= 440 && input.x <= 840 && input.y >= top && input.y <= top + 52;
71+
form.field = inside(FIELDS.user) ? "user" : inside(FIELDS.password) ? "password" : null;
72+
return inside(480);
73+
}
74+
const field = form.field;
75+
if (input.kind === "text" && field) {
76+
form[field] += input.text;
77+
} else if (input.kind === "key" && input.key === "Enter") {
78+
return true;
79+
} else if (input.kind === "key" && input.key === "Tab") {
80+
form.field = form.field === "user" ? "password" : "user";
81+
} else if (input.kind === "key" && input.key === "Backspace" && field) {
82+
form[field] = form[field].slice(0, -1);
83+
} else if (input.kind === "key" && [...input.key].length === 1 && field && input.modifiers < 2) {
84+
form[field] += input.key;
85+
}
86+
return false;
87+
}
88+
89+
export function seedHands(fake: FakeBotloft, crewId: CrewId): void {
90+
const clerk = fake.addBot(crewId, "Clerk", "Places the bakery's orders");
91+
fake.setBotState(clerk.id, "needs_approval", 2);
92+
const form: Form = { user: "", password: "", field: null, signedIn: false };
93+
const login = "https://flour.example/login";
94+
fake.browser.open(clerk.id, login, "Sign in · Flour Co.");
95+
fake.browser.frame(clerk.id, draw(form));
96+
fake.chat.tool(clerk.id, "mcp__botloft__browser_open", { summary: login, status: "done" });
97+
const task = "Sign in to Flour Co. with the bakery's account";
98+
fake.chat.tool(clerk.id, "mcp__botloft__browser_ask_owner", { summary: task, status: "running" });
99+
fake.browser.ask(clerk.id, task);
100+
fake.browser.onInput(clerk.id, (input) => {
101+
if (form.signedIn) {
102+
return;
103+
}
104+
if (handle(form, input)) {
105+
form.signedIn = true;
106+
fake.browser.open(clerk.id, "https://flour.example/orders", "Orders · Flour Co.");
107+
}
108+
fake.browser.frame(clerk.id, draw(form));
109+
});
110+
}

‎app/src/features/bots/BotHeader.tsx‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ export function BotHeader({
4040
onToggleFiles,
4141
browserOpen,
4242
browsing,
43+
asking,
4344
onToggleBrowser,
4445
screensOpen,
4546
drawing,
@@ -56,6 +57,8 @@ export function BotHeader({
5657
browserOpen: boolean;
5758
/** The bot's browser is open (spec 21.8). */
5859
browsing: boolean;
60+
/** The bot asked the owner for a hand in it (spec 21.10). */
61+
asking: boolean;
5962
onToggleBrowser(): void;
6063
screensOpen: boolean;
6164
/** The bot is writing a screen (spec 22.5). */
@@ -152,18 +155,20 @@ export function BotHeader({
152155
label={
153156
browserOpen
154157
? t.browser.hide
155-
: browsing
156-
? `${t.browser.show}: ${t.browser.browsing(bot.name)}`
157-
: t.browser.show
158+
: asking
159+
? `${t.browser.show}: ${t.browser.help.needs(bot.name)}`
160+
: browsing
161+
? `${t.browser.show}: ${t.browser.browsing(bot.name)}`
162+
: t.browser.show
158163
}
159164
aria-pressed={browserOpen}
160165
onClick={onToggleBrowser}
161166
/>
162-
{browsing && !browserOpen && (
167+
{(browsing || asking) && !browserOpen && (
163168
<span
164169
aria-hidden
165170
className="live-dot pointer-events-none absolute top-0.5 right-0.5"
166-
style={{ background: "var(--work)" }}
171+
style={{ background: asking ? "var(--warn)" : "var(--work)" }}
167172
/>
168173
)}
169174
</span>

‎app/src/features/bots/BotView.tsx‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,9 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
7676
return status === "open" || status === "starting";
7777
});
7878
const browserOpen = side === "browser";
79+
const asking = useApp((state) => Boolean(state.browsers[bot.id]?.ask));
80+
// Each ask to open the browser in the owner's hands counts up (spec 21.10).
81+
const [takeBrowser, setTakeBrowser] = useState(0);
7982
const screens = useScreens(bot);
8083
const screensOpen = side === "screens";
8184
const [screenPath, setScreenPath] = useState<string | null>(null);
@@ -106,11 +109,14 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
106109
}
107110
}
108111
}, [screens.writing]);
109-
const showBrowser = () => {
112+
const showBrowser = (options?: { take?: boolean }) => {
110113
if (filesOpen) {
111114
seen();
112115
}
113116
setSide("browser");
117+
if (options?.take) {
118+
setTakeBrowser((count) => count + 1);
119+
}
114120
};
115121
const view = stateView(bot, crew.paused, t);
116122
const tabs: Tab<Pane>[] = [
@@ -133,6 +139,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
133139
onToggleFiles={toggleFiles}
134140
browserOpen={browserOpen}
135141
browsing={browsing}
142+
asking={asking}
136143
onToggleBrowser={() => (browserOpen ? setSide(null) : showBrowser())}
137144
screensOpen={screensOpen}
138145
drawing={screens.writing !== null}
@@ -155,7 +162,7 @@ export function BotView({ bot, crew }: { bot: Bot; crew: Crew }) {
155162
)}
156163
</div>
157164
{side === "details" && <Details bot={bot} onClose={() => setSide(null)} />}
158-
{browserOpen && <BrowserPanel bot={bot} onClose={() => setSide(null)} />}
165+
{browserOpen && <BrowserPanel bot={bot} take={takeBrowser} onClose={() => setSide(null)} />}
159166
{screensOpen && (
160167
<ScreensPanel
161168
bot={bot}

‎app/src/features/browser/BrowserPanel.tsx‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
// The bot's own browser beside its chat (spec 21.8): the page it is on,
22
// live, with its cursor and a line about what it just did. Watching starts
3-
// when the panel opens and stops when it closes.
3+
// when the panel opens and stops when it closes. The owner can take it into
4+
// their own hands and give it back (spec 21.10).
45

56
import { ExternalLink, Globe, LoaderCircle, Lock, Maximize2, Minimize2, X } from "lucide-react";
6-
import { useState } from "react";
7+
import { useEffect, useRef, useState } from "react";
78
import { useT } from "../../i18n";
89
import type { Bot, BrowserState } from "../../lib/protocol.gen";
910
import { useApp, useHost } from "../../store/context";
@@ -12,17 +13,42 @@ import { Callout } from "../../ui/Callout";
1213
import { SidePanel } from "../../ui/SidePanel";
1314
import { attempt } from "../../ui/toast";
1415
import { BotAvatar } from "../bots/BotAvatar";
16+
import { AskCallout, HeldBar, TakeBar } from "./HandsBars";
17+
import { HandsLayer } from "./HandsLayer";
1518
import { LiveView, useCaption } from "./LiveView";
1619
import { useBrowserView } from "./useBrowserView";
20+
import { useHands } from "./useHands";
1721

18-
export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
22+
export function BrowserPanel({
23+
bot,
24+
take = 0,
25+
onClose,
26+
}: {
27+
bot: Bot;
28+
/** Counts up each time the owner asks to open it in their hands. */
29+
take?: number;
30+
onClose(): void;
31+
}) {
1932
const t = useT().browser;
2033
const state = useApp((app) => app.browsers[bot.id]) ?? null;
2134
const status = state?.status ?? "closed";
22-
const { frame, action } = useBrowserView(bot, true);
35+
const { frame, action, watched } = useBrowserView(bot, true);
2336
const [expanded, setExpanded] = useState(false);
2437
const caption = useCaption(bot, action);
2538
const live = status === "open" && frame !== null;
39+
const hands = useHands(bot, state);
40+
const [focused, setFocused] = useState(false);
41+
const ask = state?.ask ?? null;
42+
// Taking needs the watch first: the daemon ties the hands to it.
43+
const taken = useRef(0);
44+
useEffect(() => {
45+
if (take > taken.current && watched && status === "open") {
46+
taken.current = take;
47+
if (!hands.held) {
48+
void hands.take();
49+
}
50+
}
51+
}, [take, watched, status, hands.held, hands.take]);
2652

2753
return (
2854
<SidePanel label={t.panel(bot.name)} name="browser" defaultWidth={560} expanded={expanded}>
@@ -66,8 +92,27 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
6692
<Empty bot={bot} />
6793
) : (
6894
<>
95+
{live && ask && !hands.held && <AskCallout bot={bot} task={ask} hands={hands} />}
96+
{hands.held && <HeldBar bot={bot} task={ask} focused={focused} hands={hands} />}
6997
<div className="relative">
70-
<LiveView bot={bot} frame={frame} action={action} dim={status === "closed"} />
98+
<LiveView
99+
bot={bot}
100+
frame={frame}
101+
action={action}
102+
dim={status === "closed"}
103+
held={hands.held}
104+
>
105+
{hands.held && frame && (
106+
<HandsLayer
107+
label={t.hands.screen(bot.name)}
108+
keysLabel={t.hands.typing}
109+
width={frame.width}
110+
height={frame.height}
111+
send={hands.send}
112+
onFocus={setFocused}
113+
/>
114+
)}
115+
</LiveView>
71116
{(status !== "open" || !frame) && (
72117
<Overlay
73118
busy={status === "starting" || status === "open"}
@@ -76,7 +121,7 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
76121
/>
77122
)}
78123
</div>
79-
{status === "open" && caption && (
124+
{status === "open" && caption && !hands.held && (
80125
<p
81126
key={action?.at}
82127
aria-live="polite"
@@ -86,6 +131,7 @@ export function BrowserPanel({ bot, onClose }: { bot: Bot; onClose(): void }) {
86131
<span className="truncate">{caption}</span>
87132
</p>
88133
)}
134+
{live && !hands.held && !ask && <TakeBar bot={bot} hands={hands} />}
89135
</>
90136
)}
91137
</div>

0 commit comments

Comments
 (0)