feat(browser): teach a bot a task by doing it once

app/src/features/browser/BrowserPanel.tsx

@@ -20,6 +20,7 @@ import { ComputerDock } from "../terminal/ComputerDock";
 import { AddressBar } from "./AddressBar";
 import { AskCallout, ControlPill, HeldNotes, TakeNotes } from "./HandsBars";
 import { HandsLayer } from "./HandsLayer";
+import { LessonArea, LessonDialog, useLesson } from "./Lesson";
 import { LiveView, useCaption } from "./LiveView";
 import { TabStrip } from "./TabStrip";
 import { useBrowserView } from "./useBrowserView";
@@ -60,6 +61,7 @@ export function BrowserPanel({
   // Nobody uses it: its pages stand still until someone does (spec 21.2).
   const resting = status === "open" && state?.resting === true;
   const hands = useHands(bot, state);
+  const lesson = useLesson(bot, state);
   const [focused, setFocused] = useState(false);
   const ask = state?.ask ?? null;
   const windowed = state?.window === true;
@@ -200,7 +202,7 @@ export function BrowserPanel({
               <ComputerDock className="mt-3" />
             </div>
             {(hands.held || (live && !ask)) && (
-              <ControlPill key={String(hands.held)} bot={bot} hands={hands} />
+              <ControlPill key={String(hands.held)} bot={bot} hands={hands} lesson={lesson} />
             )}
             {status === "open" && !hands.held && (
               // Its place is kept while empty, so the page never moves for it.
@@ -213,7 +215,8 @@ export function BrowserPanel({
                 )}
               </p>
             )}
-            {hands.held && (
+            {hands.held && <LessonArea bot={bot} lesson={lesson} />}
+            {hands.held && lesson.steps === null && (
               <HeldNotes
                 bot={bot}
                 task={ask}
@@ -226,6 +229,7 @@ export function BrowserPanel({
           </>
         )}
       </div>
+      {lesson.done && <LessonDialog bot={bot} lesson={lesson} />}
     </SidePanel>
   );
 }

app/src/features/browser/HandsBars.tsx

@@ -4,13 +4,14 @@
 // it, what the owner should know, and a window of its own for sites that
 // refuse to sign in here (spec 21.11).
 
-import { Check, ChevronDown, Hand, Keyboard, MousePointerClick } from "lucide-react";
+import { Check, ChevronDown, GraduationCap, Hand, Keyboard, MousePointerClick } from "lucide-react";
 import { useState } from "react";
 import { useT } from "../../i18n";
 import type { Bot } from "../../lib/protocol.gen";
 import { Button } from "../../ui/Button";
 import { Callout } from "../../ui/Callout";
 import { BotAvatar, moodOf } from "../bots/BotAvatar";
+import type { Lesson } from "./Lesson";
 import type { Hands } from "./useHands";
 import { WindowButton } from "./WindowBar";
 
@@ -43,9 +44,20 @@ export function AskCallout({ bot, task, hands }: { bot: Bot; task: string; hands
  * Who is in control, hanging from the bottom of the live screen: the bot,
  * with the button to take it, or the owner, with the one to give it back.
  */
-export function ControlPill({ bot, hands }: { bot: Bot; hands: Hands }) {
-  const t = useT().browser.hands;
+export function ControlPill({
+  bot,
+  hands,
+  lesson,
+}: {
+  bot: Bot;
+  hands: Hands;
+  /** While the owner teaches, the pill ends the lesson first (spec 21.13). */
+  lesson?: Lesson;
+}) {
+  const words = useT().browser;
+  const t = words.hands;
   const held = hands.held;
+  const teaching = held && lesson?.steps != null;
   return (
     <div
       role="status"
@@ -64,12 +76,14 @@ export function ControlPill({ bot, hands }: { bot: Bot; hands: Hands }) {
       <Button
         variant={held ? "primary" : "secondary"}
         size="sm"
-        icon={held ? Check : Hand}
+        icon={teaching ? GraduationCap : held ? Check : Hand}
         disabled={hands.busy}
-        onClick={() => void (held ? hands.release() : hands.take())}
+        onClick={() => void (teaching ? lesson?.finish() : held ? hands.release() : hands.take())}
         className="min-w-0 shrink! rounded-full! px-3!"
       >
-        <span className="truncate">{held ? t.giveBack(bot.name) : t.take}</span>
+        <span className="truncate">
+          {teaching ? words.lesson.finish : held ? t.giveBack(bot.name) : t.take}
+        </span>
       </Button>
     </div>
   );

app/src/features/browser/Lesson.tsx

@@ -0,0 +1,180 @@
+// Teaching the bot a task (spec 21.13): with the browser in the owner's
+// hands, a lesson records what they do as steps by what it means; at its
+// end they name the task and make it a routine, or send it for the bot to
+// remember.
+
+import { AlarmClock, GraduationCap, Send, X } from "lucide-react";
+import { type FormEvent, useState } from "react";
+import { useT } from "../../i18n";
+import type { Bot, BrowserState, LessonStep } from "../../lib/protocol.gen";
+import { useApi } from "../../store/context";
+import { Button } from "../../ui/Button";
+import { Dialog } from "../../ui/Dialog";
+import { TextField } from "../../ui/Field";
+import { attempt } from "../../ui/toast";
+import { RoutineDialog } from "../routines/RoutineDialog";
+import { lessonText, stepText } from "./lessonText";
+
+export interface Lesson {
+  /** The steps so far, while the owner teaches; `null` otherwise. */
+  steps: LessonStep[] | null;
+  start(): Promise<void>;
+  /** Ends the lesson and opens what to do with it. */
+  finish(): Promise<void>;
+  cancel(): Promise<void>;
+  /** The steps of the lesson just finished, until the owner is done with them. */
+  done: LessonStep[] | null;
+  close(): void;
+}
+
+export function useLesson(bot: Pick<Bot, "id">, state: BrowserState | null): Lesson {
+  const api = useApi();
+  const t = useT().browser.lesson;
+  const [done, setDone] = useState<LessonStep[] | null>(null);
+  const teach = async (on: boolean) => {
+    let steps: LessonStep[] = [];
+    const ok = await attempt(t.failed, async () => {
+      steps = await api.call("browser.teach", { botId: bot.id, on });
+    });
+    return ok ? steps : null;
+  };
+  return {
+    steps: state?.lesson ?? null,
+    start: async () => {
+      await teach(true);
+    },
+    finish: async () => {
+      const steps = await teach(false);
+      if (steps) {
+        setDone(steps);
+      }
+    },
+    cancel: async () => {
+      await teach(false);
+    },
+    done,
+    close: () => setDone(null),
+  };
+}
+
+/** Under the owner's pill: start a lesson, or the one being given, live. */
+export function LessonArea({ bot, lesson }: { bot: Bot; lesson: Lesson }) {
+  const t = useT().browser.lesson;
+  if (lesson.steps === null) {
+    return (
+      <div className="mt-2 flex flex-col items-center gap-1 text-center">
+        <Button size="sm" icon={GraduationCap} onClick={() => void lesson.start()}>
+          {t.teach}
+        </Button>
+        <p className="max-w-md text-muted text-xs">{t.teachWhy(bot.name)}</p>
+      </div>
+    );
+  }
+  return (
+    <section
+      aria-label={t.recording}
+      className="mx-auto mt-3 flex w-full max-w-md animate-rise flex-col gap-2 rounded-xl border border-danger/40 bg-panel px-3.5 py-3"
+    >
+      <p className="flex items-center gap-2 font-medium text-sm">
+        <span aria-hidden className="live-dot" />
+        {t.recording}
+      </p>
+      {lesson.steps.length === 0 ? (
+        <p className="text-muted text-xs">{t.empty}</p>
+      ) : (
+        <ol className="flex max-h-40 flex-col gap-1 overflow-y-auto text-ink-soft text-xs">
+          {lesson.steps.map((step, index) => (
+            // biome-ignore lint/suspicious/noArrayIndexKey: steps only grow at the end
+            <li key={index} className="animate-rise">
+              {index + 1}. {stepText(step, t)}
+            </li>
+          ))}
+        </ol>
+      )}
+      {/* The pill above finishes it; here it can be dropped. */}
+      <div className="flex">
+        <Button size="sm" variant="ghost" onClick={() => void lesson.cancel()}>
+          {t.cancel}
+        </Button>
+      </div>
+    </section>
+  );
+}
+
+/** What to do with the lesson just finished: a routine, or a message. */
+export function LessonDialog({ bot, lesson }: { bot: Bot; lesson: Lesson }) {
+  const t = useT().browser.lesson;
+  const api = useApi();
+  const [name, setName] = useState("");
+  const [steps, setSteps] = useState(lesson.done ?? []);
+  const [routine, setRoutine] = useState(false);
+  const ready = name.trim().length > 0 && steps.length > 0;
+  const text = lessonText(name, steps, t);
+
+  const send = async (event?: FormEvent) => {
+    event?.preventDefault();
+    if (!ready) return;
+    const body = `${text}\n\n${t.forMemory(name.trim())}`;
+    // It shows in the bot's chat, beside the panel.
+    if (await attempt(t.sendFailed, () => api.call("messages.send", { botId: bot.id, body }))) {
+      lesson.close();
+    }
+  };
+
+  if (routine) {
+    return (
+      <RoutineDialog
+        bot={bot}
+        start={{ name: name.trim(), prompt: `${text}\n\n${t.forRoutine}` }}
+        onClose={lesson.close}
+      />
+    );
+  }
+  return (
+    <Dialog title={t.title(bot.name)} onClose={lesson.close}>
+      <form onSubmit={send} className="flex flex-col gap-4">
+        <TextField
+          label={t.name}
+          value={name}
+          placeholder={t.namePlaceholder}
+          onChange={(event) => setName(event.target.value)}
+          autoFocus
+        />
+        <div>
+          <p className="mb-1.5 font-medium text-sm">{t.steps}</p>
+          <ol className="flex max-h-64 flex-col gap-1 overflow-y-auto">
+            {steps.map((step, index) => (
+              // biome-ignore lint/suspicious/noArrayIndexKey: a list the owner only shortens
+              <li key={index} className="flex items-center gap-2 text-ink-soft text-sm">
+                <span className="w-5 shrink-0 text-right text-muted text-xs">{index + 1}.</span>
+                <span className="min-w-0 flex-1 truncate" title={stepText(step, t)}>
+                  {stepText(step, t)}
+                </span>
+                <Button
+                  size="sm"
+                  variant="ghost"
+                  icon={X}
+                  label={t.remove(index + 1)}
+                  onClick={() => setSteps(steps.filter((_, at) => at !== index))}
+                />
+              </li>
+            ))}
+          </ol>
+        </div>
+        <div className="flex flex-wrap justify-end gap-2">
+          <Button icon={Send} type="submit" disabled={!ready}>
+            {t.sendToBot(bot.name)}
+          </Button>
+          <Button
+            variant="primary"
+            icon={AlarmClock}
+            disabled={!ready}
+            onClick={() => setRoutine(true)}
+          >
+            {t.makeRoutine}
+          </Button>
+        </div>
+      </form>
+    </Dialog>
+  );
+}

app/src/features/browser/lesson.test.tsx

@@ -0,0 +1,113 @@
+import { act, cleanup, fireEvent, screen, waitFor, within } from "@testing-library/react";
+import { afterEach, describe, expect, test } from "vitest";
+import { en } from "../../i18n/en";
+import { FakeBotloft } from "../../lib/fake";
+import { crewOpened, openBot, renderApp } from "../../test/app";
+import { lessonText } from "./lessonText";
+
+afterEach(cleanup);
+
+/** Scout's browser open on a sign-in page, in the owner's hands. */
+async function inHands() {
+  const fake = new FakeBotloft();
+  const ops = fake.addCrew("Ops");
+  const scout = fake.addBot(ops.id, "Scout", "Finds sources");
+  fake.setBotState(scout.id, "busy");
+  fake.browser.open(scout.id, "https://shop.example/login?next=/orders", "Sign in");
+  renderApp(fake);
+  await crewOpened("Ops");
+  openBot("Scout");
+  await screen.findByRole("list", { name: "Messages" });
+  fireEvent.click(screen.getByRole("button", { name: /^Show browser/ }));
+  await waitFor(() => expect(fake.browser.watching).toBe(scout.id));
+  act(() => {
+    fake.browser.frame(scout.id, "AAAA");
+  });
+  fireEvent.click(screen.getByRole("button", { name: "Take control" }));
+  await screen.findByText("You are in control");
+  return { fake, scout };
+}
+
+/** The daemon reads what the owner did as steps. */
+function teachSteps(fake: FakeBotloft, botId: string) {
+  act(() => {
+    fake.lesson.step(botId, { kind: "click", label: "E-mail", role: "textbox" });
+    fake.lesson.step(botId, { kind: "type", label: "E-mail", role: "textbox" });
+    fake.lesson.step(botId, { kind: "type", label: "Password", role: "textbox", secret: true });
+    fake.lesson.step(botId, { kind: "press", label: "Enter" });
+  });
+}
+
+async function finish(fake: FakeBotloft, botId: string) {
+  fireEvent.click(screen.getByRole("button", { name: "Teach a task" }));
+  const recording = await screen.findByRole("region", { name: "Recording the lesson" });
+  expect(within(recording).getByText("1. Open https://shop.example/login")).toBeDefined();
+  teachSteps(fake, botId);
+  expect(
+    within(recording).getByText('4. Type the password in "Password": ask me to type it'),
+  ).toBeDefined();
+  // The pill ends the lesson before giving anything back.
+  fireEvent.click(screen.getByRole("button", { name: "Finish the lesson" }));
+  const dialog = await screen.findByRole("dialog", { name: "Teach Scout a task" });
+  fireEvent.change(within(dialog).getByLabelText("Name of the task"), {
+    target: { value: "Check my orders" },
+  });
+  return dialog;
+}
+
+describe("teaching a task", () => {
+  test("the words say where things were typed, never what", () => {
+    const text = lessonText(
+      "Check my orders",
+      [
+        { kind: "open", label: "https://shop.example/login", role: null, secret: false },
+        { kind: "type", label: "E-mail", role: "textbox", secret: false },
+        { kind: "type", label: "Password", role: "textbox", secret: true },
+      ],
+      en.browser.lesson,
+    );
+    expect(text).toBe(
+      [
+        'How to do "Check my orders" in the browser, as I showed you:',
+        "1. Open https://shop.example/login",
+        '2. Type in "E-mail"',
+        '3. Type the password in "Password": ask me to type it',
+        "",
+        "Where I typed something, I don't show what: use what the task needs, or ask me.",
+      ].join("\n"),
+    );
+  });
+
+  test("a lesson records live and goes to the bot to remember, without the steps taken out", async () => {
+    const { fake, scout } = await inHands();
+    const dialog = await finish(fake, scout.id);
+    fireEvent.click(within(dialog).getByRole("button", { name: "Remove step 2" }));
+    fireEvent.click(within(dialog).getByRole("button", { name: "Send to Scout to remember" }));
+    await waitFor(() =>
+      expect(fake.calls.some((call) => call.method === "messages.send")).toBe(true),
+    );
+    const sent = fake.calls.find((call) => call.method === "messages.send")?.params as {
+      body: string;
+    };
+    expect(sent.body).toContain('How to do "Check my orders"');
+    expect(sent.body).toContain('2. Type in "E-mail"');
+    expect(sent.body).not.toContain('Click "E-mail"');
+    expect(sent.body).toContain('so you can do "Check my orders" when I ask.');
+    await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull());
+    // The owner still has the browser.
+    expect(screen.getByText("You are in control")).toBeDefined();
+  });
+
+  test("a lesson becomes a routine, its request already written", async () => {
+    const { fake, scout } = await inHands();
+    const dialog = await finish(fake, scout.id);
+    fireEvent.click(within(dialog).getByRole("button", { name: "Make it a routine…" }));
+    const routine = await screen.findByRole("dialog", { name: "New routine for Scout" });
+    expect((within(routine).getByLabelText("Name") as HTMLInputElement).value).toBe(
+      "Check my orders",
+    );
+    const prompt = within(routine).getByLabelText("What should Scout do?") as HTMLTextAreaElement;
+    expect(prompt.value).toContain("1. Open https://shop.example/login");
+    expect(prompt.value.endsWith("Do it now.")).toBe(true);
+  });
+});