feat(app): the bot's cursor follows each part of a screen as it is written

While a bot writes a screen, its cursor (the arrow in its color with its
name, shared with the browser as BotCursor) glides to the end of what is
written, and a frame in its color marks the part being built, on the
artboard and on the screen in focus. Only the frame on show counts, so
the cursor matches what is seen; measures are scaled by the view the
script saw. With nothing visible yet the cursor waits at the top; near
the right or bottom edge its name moves aside. The preview's drafts use
the daemon's own script, and it writes at a pace closer to a model's.

Author: httpsphl

app/src/dev/seedScreens.ts

@@ -61,7 +61,7 @@ export function seedScreens(fake: FakeBotloft, crewId: CrewId): void {
       }
       return;
     }
-    written = Math.min(LANDING.length, written + 70);
+    written = Math.min(LANDING.length, written + 24);
     const done = written === LANDING.length;
     if (done) {
       fake.screens.add(designer.id, "index.html", LANDING, { folder: "site" });

app/src/features/bots/BotCursor.tsx

@@ -0,0 +1,63 @@
+// A bot's cursor: an arrow in its color with its name, gliding to where it
+// works. The browser shows it where the bot clicks (spec 21.8), the design
+// area where the bot is writing a screen (spec 22.5).
+
+import type { Bot } from "../../lib/protocol.gen";
+
+/** Dark text on light colors, light text on dark ones. */
+export function inkOn(color: string): string {
+  const hex = color.replace("#", "");
+  const [r, g, b] = [0, 2, 4].map((at) => Number.parseInt(hex.slice(at, at + 2), 16) / 255);
+  const luminance = 0.2126 * (r ?? 0) + 0.7152 * (g ?? 0) + 0.0722 * (b ?? 0);
+  return luminance > 0.55 ? "#141414" : "#ffffff";
+}
+
+export function BotCursor({
+  bot,
+  left,
+  top,
+  typing = false,
+  flip = false,
+  lift = false,
+}: {
+  bot: Pick<Bot, "name" | "color">;
+  /** Where the tip points, as CSS lengths inside the positioned parent. */
+  left: string;
+  top: string;
+  /** Dots after the name: the bot is writing. */
+  typing?: boolean;
+  /** The name goes left of the arrow, near the right edge. */
+  flip?: boolean;
+  /** The name goes above the arrow, near the bottom edge. */
+  lift?: boolean;
+}) {
+  return (
+    <span
+      aria-hidden
+      className={`bot-cursor ${flip ? "flex-row-reverse" : ""}`}
+      style={{ left, top, ...(flip && { translate: "calc(-100% + 18px) -3px" }) }}
+    >
+      <svg width="22" height="22" viewBox="0 0 24 24" className="shrink-0 drop-shadow-sm">
+        <title>{bot.name}</title>
+        <path
+          d="M4 2.5 19.5 12l-7 1.6-3.4 6.9Z"
+          fill={bot.color}
+          stroke="#ffffff"
+          strokeWidth="1.6"
+          strokeLinejoin="round"
+        />
+      </svg>
+      <span
+        className={`${flip ? "mr-1" : "ml-3.5"} -mt-1 inline-flex items-center gap-1 whitespace-nowrap rounded-full px-2 py-0.5 font-semibold text-[11px] shadow-sm`}
+        style={{
+          background: bot.color,
+          color: inkOn(bot.color),
+          ...(lift && { marginTop: "-20px" }),
+        }}
+      >
+        {bot.name}
+        {typing && <span className="bot-typing">•••</span>}
+      </span>
+    </span>
+  );
+}

app/src/features/browser/LiveView.tsx

@@ -5,17 +5,10 @@
 import { type ReactNode, useEffect, useState } from "react";
 import { useT } from "../../i18n";
 import type { Bot, BrowserAction, BrowserFrame } from "../../lib/protocol.gen";
+import { BotCursor } from "../bots/BotCursor";
 
 const PAGE = { width: 1280, height: 800 };
 
-/** Dark text on light colors, light text on dark ones. */
-function inkOn(color: string): string {
-  const hex = color.replace("#", "");
-  const [r, g, b] = [0, 2, 4].map((at) => Number.parseInt(hex.slice(at, at + 2), 16) / 255);
-  const luminance = 0.2126 * (r ?? 0) + 0.7152 * (g ?? 0) + 0.0722 * (b ?? 0);
-  return luminance > 0.55 ? "#141414" : "#ffffff";
-}
-
 /** Where on the screen, in percent, an action happened. */
 function spot(action: BrowserAction | null, frame: BrowserFrame | null) {
   if (!action || action.x === null || action.y === null) {
@@ -83,7 +76,6 @@ export function LiveView({
   }, [action?.at, action?.kind]);
   const width = frame?.width ?? PAGE.width;
   const height = frame?.height ?? PAGE.height;
-  const ink = inkOn(bot.color);
   const acting = !dim && !held;
 
   return (
@@ -113,29 +105,12 @@ export function LiveView({
         />
       )}
       {acting && cursor && (
-        <span
-          aria-hidden
-          className="browser-cursor"
-          style={{ left: `${cursor.x}%`, top: `${cursor.y}%` }}
-        >
-          <svg width="22" height="22" viewBox="0 0 24 24" className="drop-shadow-sm">
-            <title>{bot.name}</title>
-            <path
-              d="M4 2.5 19.5 12l-7 1.6-3.4 6.9Z"
-              fill={bot.color}
-              stroke="#ffffff"
-              strokeWidth="1.6"
-              strokeLinejoin="round"
-            />
-          </svg>
-          <span
-            className="ml-3.5 -mt-1 inline-flex items-center gap-1 whitespace-nowrap rounded-full px-2 py-0.5 font-semibold text-[11px] shadow-sm"
-            style={{ background: bot.color, color: ink }}
-          >
-            {bot.name}
-            {action?.kind === "type" && <span className="browser-typing">•••</span>}
-          </span>
-        </span>
+        <BotCursor
+          bot={bot}
+          left={`${cursor.x}%`}
+          top={`${cursor.y}%`}
+          typing={action?.kind === "type"}
+        />
       )}
       {children}
     </figure>

app/src/features/browser/browser.test.tsx

@@ -54,7 +54,7 @@ describe("browser panel", () => {
     const screenshot = within(panel()).getByRole("figure", { name: "What Scout sees" });
     const image = screenshot.querySelector("img");
     expect(image?.getAttribute("src")).toBe("data:image/jpeg;base64,AAAA");
-    const cursor = screenshot.querySelector(".browser-cursor") as HTMLElement;
+    const cursor = screenshot.querySelector(".bot-cursor") as HTMLElement;
     expect(cursor.style.left).toBe("50%");
     expect(cursor.style.top).toBe("50%");
     expect(within(panel()).getByText("Live")).toBeDefined();

app/src/features/screens/LiveFrame.tsx

@@ -1,16 +1,23 @@
 // A screen drawn at its device's real size and scaled to fit (spec 22.5).
 // Each new version loads in a hidden frame and takes the front once it is
-// ready, so a screen being written grows without flickering.
+// ready, so a screen being written grows without flickering. While the bot
+// writes it, its cursor follows the end of the page and a frame marks the
+// part being built (spec 22.3).
 
-import { useEffect, useReducer } from "react";
-import type { ScreenDevice } from "../../lib/protocol.gen";
+import { useEffect, useReducer, useRef, useState } from "react";
+import type { Bot, ScreenDevice } from "../../lib/protocol.gen";
+import { BotCursor } from "../bots/BotCursor";
+import { type Mark, readMark } from "./cursorMark";
 
 export const DEVICES: Record<ScreenDevice, { width: number; height: number }> = {
   desktop: { width: 1280, height: 800 },
   tablet: { width: 834, height: 1112 },
   mobile: { width: 390, height: 844 },
 };
 
+/** Where the cursor waits while nothing on the page shows yet. */
+const PARKED = { x: 40, y: 40 };
+
 /** The screen runs its own scripts, away from the app (spec 22.6). */
 const SANDBOX = "allow-scripts allow-forms allow-popups allow-modals";
 
@@ -53,13 +60,16 @@ export function LiveFrame({
   scale,
   title,
   interactive = false,
+  writer = null,
 }: {
   url: string;
   device: ScreenDevice;
   scale: number;
   title: string;
   /** Takes clicks and scrolling; otherwise the frame is only a picture. */
   interactive?: boolean;
+  /** The bot writing it now, whose cursor shows. */
+  writer?: Pick<Bot, "name" | "color"> | null;
 }) {
   const [state, change] = useReducer(frames, {
     urls: [url, null],
@@ -69,6 +79,30 @@ export function LiveFrame({
   });
   useEffect(() => change({ kind: "show", url }), [url]);
   const size = DEVICES[device];
+  const windows = useRef<(Window | null)[]>([null, null]);
+  // Each frame's own mark: the one behind speaks before it is on show, and
+  // its layout is the next version's, so only the front one's shows.
+  const [marks, setMarks] = useState<[Mark | null, Mark | null]>([null, null]);
+  const writing = writer !== null;
+  useEffect(() => {
+    if (!writing) {
+      setMarks([null, null]);
+      return;
+    }
+    const listen = (event: MessageEvent) => {
+      const frame = event.source === null ? -1 : windows.current.indexOf(event.source as Window);
+      const read = frame === 0 || frame === 1 ? readMark(event.data, size) : undefined;
+      if (read) {
+        setMarks((current) => (frame === 0 ? [read, current[1]] : [current[0], read]));
+      }
+    };
+    window.addEventListener("message", listen);
+    return () => window.removeEventListener("message", listen);
+  }, [writing, size]);
+  const mark = marks[state.front];
+  const box = writing ? mark?.box : null;
+  // Nothing to see yet (the styles come first): the cursor waits at the top.
+  const point = writing && mark ? (mark.point ?? PARKED) : null;
 
   return (
     <div
@@ -84,6 +118,9 @@ export function LiveFrame({
         return (
           <iframe
             key={frame}
+            ref={(node) => {
+              windows.current[frame] = node?.contentWindow ?? null;
+            }}
             src={src}
             title={title}
             sandbox={SANDBOX}
@@ -102,6 +139,30 @@ export function LiveFrame({
           />
         );
       })}
+      {box && writer && (
+        <span
+          aria-hidden
+          className="pointer-events-none absolute rounded-[4px] border-2 transition-[left,top,width,height] duration-500 ease-out"
+          style={{
+            left: box.x * scale - 3,
+            top: box.y * scale - 3,
+            width: box.width * scale + 6,
+            height: box.height * scale + 6,
+            borderColor: writer.color,
+            boxShadow: `0 0 0 4px ${writer.color}26`,
+          }}
+        />
+      )}
+      {point && writer && (
+        <BotCursor
+          bot={writer}
+          left={`${point.x * scale}px`}
+          top={`${point.y * scale}px`}
+          typing
+          flip={point.x > size.width * 0.7}
+          lift={point.y > size.height * 0.88}
+        />
+      )}
     </div>
   );
 }

app/src/features/screens/ScreenFocus.tsx

@@ -92,6 +92,7 @@ export function ScreenFocus({
             scale={scale}
             title={screen.name}
             interactive
+            writer={screen.writing ? bot : null}
           />
         </div>
       </div>

app/src/features/screens/ScreensPanel.tsx

@@ -233,7 +233,13 @@ function Artboard({
             : undefined
         }
       >
-        <LiveFrame url={screen.url} device={device} scale={scale} title={screen.name} />
+        <LiveFrame
+          url={screen.url}
+          device={device}
+          scale={scale}
+          title={screen.name}
+          writer={screen.writing ? bot : null}
+        />
         <button
           type="button"
           aria-label={t.open(screen.name)}