From fc560cda94f0b6ebcbdaa9cff57fc26fa97134cf Mon Sep 17 00:00:00 2001 From: scottkalw Date: Sun, 15 Nov 2026 05:42:46 -0600 Subject: [PATCH 1/4] Recorder draws a cursor that glides and clicks, a box round what a say is about, and types slower A say may carry at/label/style; on the mark the overlay outlines the thing (or spotlights it) with a label tab, and marks.json gains box, label, selector and style. A say whose at matches nothing refuses the take with its step number. click, fill and choose glide a drawn cursor to the target and start a ripple as the press is sent; their marks gain cursor_from and ripple_at, and fill/choose gain x, y. Typing defaults to 0.085 s a key and a scene may set type_delay_seconds. Co-Authored-By: Claude Opus 5.5 Signed-off-by: scottkalw --- scripts/record-scene.py | 292 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 275 insertions(+), 17 deletions(-) diff --git a/scripts/record-scene.py b/scripts/record-scene.py index e9e6b1bb..62a50518 100644 --- a/scripts/record-scene.py +++ b/scripts/record-scene.py @@ -222,9 +222,35 @@ # the planned length has to include them or a scene that fits on paper runs over. SMALL_STEP_SECONDS = 0.4 SCROLL_SECONDS = 0.5 -# How long `fill` waits between the keys it types: slow enough to be seen -# typed in a 25-frame video, quick enough that a routing code is not a wait. -TYPE_DELAY_SECONDS = 0.06 +# How long `fill` waits between the keys it types, unless the scene says +# `type_delay_seconds`: about twelve characters a second, which a viewer reads +# as it arrives. The first five classes typed at 0.06 s a key, and a routing +# code went by faster than the eye could follow it (Scott, 2026-11-15). A +# scene may type slower; never faster than TYPE_DELAY_FLOOR_SECONDS. +TYPE_DELAY_SECONDS = 0.085 +TYPE_DELAY_FLOOR_SECONDS = 0.05 + +# The drawn cursor. On every `click`, `fill` and `choose` it glides from where +# it is to the middle of the thing, rests a moment, and the press is sent as a +# ring spreads from the point and fades, so the frame and the action agree. A +# viewer who sees only the screen cannot tell what was pressed, or that +# anything was (Scott, 2026-11-15: "Just the screen doesn't explain anything"). +CURSOR_GLIDE_SECONDS = 0.5 +CURSOR_REST_SECONDS = 0.15 +RIPPLE_SECONDS = 0.6 +# After a click the cursor follows the thing to where the page settled it. +CURSOR_FOLLOW_SECONDS = 0.15 + +# The box a `say` draws round what its line is about (`at`), with its `label` +# on a tab: a cyan that reads on the dark screens and is none of the state +# pills' green, amber or red, nor the badge's yellow. `spotlight` dims the +# rest of the frame instead of only outlining. The box stands this far out +# from the thing, so the outline does not sit on its text. +CALLOUT_COLOUR = "#22d3ee" +CALLOUT_STYLES = ("box", "spotlight") +CALLOUT_PAD = 6 +#: How long a `say` looks for the thing its `at` names before the take is refused. +CALLOUT_FIND_SECONDS = 2.0 # --------------------------------------------------------------- the scene @@ -261,6 +287,11 @@ def load_scene(path: Path) -> dict: scene["lesson"] = str(scene["lesson"]) if scene["take_is"] not in ("lesson", "teaser"): raise SystemExit(f"{path}: `take_is` is \"lesson\" or \"teaser\"") + delay = scene.setdefault("type_delay_seconds", TYPE_DELAY_SECONDS) + if isinstance(delay, bool) or not isinstance(delay, (int, float)) or delay < TYPE_DELAY_FLOOR_SECONDS: + raise SystemExit(f"{path}: `type_delay_seconds` is the seconds between typed keys, at least" + f" {TYPE_DELAY_FLOOR_SECONDS:g} — faster than that is not read as it is typed") + scene["type_delay_seconds"] = float(delay) part = scene["part"] for number, step in enumerate(steps, start=1): @@ -287,6 +318,7 @@ def load_scene(path: Path) -> dict: if not step.get("line"): raise SystemExit(f"{where}: a `say` needs a `line` — it is the sentence on screen") step.setdefault("hold", hold_for_words(step["line"])) + check_pointer(step, where) if verb == "slide": if not step.get("name") or not step.get("title"): raise SystemExit(f"{where}: a `slide` needs a `name` (it names the card's file) and a `title`") @@ -322,6 +354,28 @@ def load_scene(path: Path) -> dict: return scene +def check_pointer(step: dict, where: str) -> None: + """Refuse a `say`'s `at`, `label` or `style` this script would have to guess at. + + `at` is the selector of the thing the line is about; `label` the few words + on the box's tab; `style` `box` (the default) or `spotlight`. A label or a + style with nothing to point at is refused: it would be drawn nowhere. + """ + if "at" not in step: + for key in ("label", "style"): + if key in step: + raise SystemExit(f"{where}: `{key}` belongs to a `say` that points `at` something," + " and this one does not") + return + if not isinstance(step["at"], str) or not step["at"].strip(): + raise SystemExit(f"{where}: `at` is the selector of the thing the line is about") + if not isinstance(step.get("label", ""), str): + raise SystemExit(f"{where}: `label` is the few words on the box's tab") + step.setdefault("style", "box") + if step["style"] not in CALLOUT_STYLES: + raise SystemExit(f"{where}: `style = {step['style']!r}` is not one of {', '.join(CALLOUT_STYLES)}") + + def check_command_card(step: dict, where: str, scene: dict) -> None: """Refuse a terminal card this script would have to guess at, or one that could write. @@ -407,7 +461,10 @@ def planned_seconds(scene: dict) -> float: if step["do"] == "goto": total += float(step["settle"]) elif step["do"] == "fill": - total += len(step["text"]) * TYPE_DELAY_SECONDS + SMALL_STEP_SECONDS + total += (len(step["text"]) * float(scene.get("type_delay_seconds", TYPE_DELAY_SECONDS)) + + SMALL_STEP_SECONDS + CURSOR_GLIDE_SECONDS + CURSOR_REST_SECONDS) + elif step["do"] in ("click", "choose"): + total += SMALL_STEP_SECONDS + CURSOR_GLIDE_SECONDS + CURSOR_REST_SECONDS elif step["do"] == "say": total += float(step["hold"]) elif step["do"] == "hold": @@ -450,7 +507,9 @@ def play(scene: dict, stage) -> None: elif verb == "wait_for": stage.wait_for(step, float(step["timeout"])) elif verb == "say": - stage.say(step["line"], float(step["hold"])) + # A `say` that points at nothing is handed exactly what it always was. + pointer = {key: step[key] for key in ("at", "label", "style") if key in step} + stage.say(step["line"], float(step["hold"]), **pointer) elif verb == "hold": stage.hold(float(step["seconds"])) elif verb == "mark": @@ -465,6 +524,14 @@ def play(scene: dict, stage) -> None: # --------------------------------------------------------------- the overlay +#: The drawn cursor: a plain arrow, white with a dark edge, its tip at (2, 2). +CURSOR_SVG = ( + "" + "" +) + + def overlay_script(badge: str, captions: bool = True) -> str: """The caption bar, the badge, and the two buttons that are hidden. @@ -488,6 +555,15 @@ def overlay_script(badge: str, captions: bool = True) -> str: And the product's floating Assistant and Design buttons are hidden, because they sit exactly where the caption bar goes. + + It also draws the cursor, the ripple of a press, and the box a `say` puts + round what it is about (`__sceneCursorTo`, `__sceneRipple`, + `__sceneCallout`). All three sit under the caption bar and the badge, so + neither is ever drawn over, and none of them takes a press: the page's own + controls are what Playwright presses. The cursor's place is kept in + `sessionStorage` like the caption, so a click that leaves the page finds + the cursor where it was on the next. A card (a `slide`) is the recorder's + own picture and nothing is pressed on it: it has no cursor. """ css = ( f"#fs-scene-cap{{position:fixed!important;left:0!important;right:{CAPTION_RIGHT_INSET}px!important;" @@ -504,6 +580,21 @@ def overlay_script(badge: str, captions: bool = True) -> str: "padding:0 18px!important;border-radius:8px!important;white-space:nowrap!important;" "box-shadow:0 2px 10px rgba(0,0,0,.5)!important}" ".assist-launch,.design-launch{display:none!important}" + # The cursor's tip is the point, at the element's top left; it moves by transform. + "#fs-scene-cursor{position:fixed!important;left:0!important;top:0!important;width:26px!important;" + "height:30px!important;margin:-2px 0 0 -2px!important;pointer-events:none!important;" + "z-index:2147483645!important;filter:drop-shadow(0 2px 3px rgba(0,0,0,.6))!important}" + ".fs-scene-ripple{position:fixed!important;width:56px!important;height:56px!important;" + f"border:3px solid {CALLOUT_COLOUR}!important;border-radius:50%!important;box-sizing:border-box!important;" + "pointer-events:none!important;z-index:2147483645!important;background:rgba(34,211,238,.18)!important}" + "#fs-scene-callout{position:fixed!important;box-sizing:border-box!important;pointer-events:none!important;" + f"border:3px solid {CALLOUT_COLOUR}!important;border-radius:8px!important;z-index:2147483643!important;" + "box-shadow:0 0 0 1px rgba(0,0,0,.55),0 0 14px rgba(34,211,238,.45)!important}" + "#fs-scene-callout.fs-spot{box-shadow:0 0 0 1px rgba(0,0,0,.55),0 0 0 200vmax rgba(4,7,11,.62)!important}" + "#fs-scene-callout-label{position:fixed!important;pointer-events:none!important;z-index:2147483644!important;" + f"background:{CALLOUT_COLOUR}!important;color:#04141a!important;white-space:nowrap!important;" + "font:700 17px/1.25 system-ui,-apple-system,Segoe UI,sans-serif!important;padding:4px 11px!important;" + "border-radius:7px!important;box-shadow:0 2px 8px rgba(0,0,0,.5)!important}" # The product's message after a save sits 22 px off the bottom, under # the caption bar; it is lifted above the bar's two lines, where the # viewer can read it, and is otherwise drawn as the product draws it. @@ -525,6 +616,83 @@ def overlay_script(badge: str, captions: bool = True) -> str: cap.id = 'fs-scene-cap'; cap.textContent = remembered(); root.append(style, badge, cap); + if (!location.pathname.startsWith({json.dumps(SLIDE_PATH)})) {{ + const cursor = document.createElement('div'); + cursor.id = 'fs-scene-cursor'; + cursor.innerHTML = {json.dumps(CURSOR_SVG)}; + const at = cursorAt(); + cursor.style.transform = `translate(${{at.x}}px, ${{at.y}}px)`; + root.append(cursor); + }} + }}; + const CURSOR = 'fsSceneCursor'; + function cursorAt() {{ + try {{ + const kept = JSON.parse(sessionStorage.getItem(CURSOR) || 'null'); + if (kept && Number.isFinite(kept.x) && Number.isFinite(kept.y)) return kept; + }} catch (e) {{ /* the middle of the window, then */ }} + return {{ x: Math.round(innerWidth / 2), y: Math.round(innerHeight / 2) }}; + }} + // Glide the cursor to (x, y) over `ms`, and answer where it came from once it is there. + window.__sceneCursorTo = (x, y, ms) => new Promise((resolve) => {{ + draw(); + const from = cursorAt(); + try {{ sessionStorage.setItem(CURSOR, JSON.stringify({{ x, y }})); }} catch (e) {{ /* not worth a take */ }} + const cursor = document.getElementById('fs-scene-cursor'); + if (!cursor) return resolve(null); + cursor.style.transition = ms > 0 ? `transform ${{ms}}ms cubic-bezier(.3,.1,.2,1)` : 'none'; + cursor.style.transform = `translate(${{x}}px, ${{y}}px)`; + setTimeout(() => resolve(from), ms); + }}); + window.__sceneRipple = (x, y) => {{ + const ring = document.createElement('div'); + ring.className = 'fs-scene-ripple'; + ring.style.left = (x - 28) + 'px'; + ring.style.top = (y - 28) + 'px'; + document.documentElement.append(ring); + ring.animate([{{ transform: 'scale(.25)', opacity: 1 }}, {{ transform: 'scale(1.4)', opacity: 0 }}], + {{ duration: {int(RIPPLE_SECONDS * 1000)}, easing: 'ease-out' }}).onfinish = () => ring.remove(); + }}; + // The box round what a line is about, following it if the page moves it, until cleared. + let callout = null; + window.__sceneCallout = (el, label, style) => {{ + window.__sceneCalloutClear(); + const box = document.createElement('div'); + box.id = 'fs-scene-callout'; + if (style === 'spotlight') box.classList.add('fs-spot'); + const tab = label ? document.createElement('div') : null; + if (tab) {{ tab.id = 'fs-scene-callout-label'; tab.textContent = label; }} + document.documentElement.append(box, ...(tab ? [tab] : [])); + const pad = {CALLOUT_PAD}; + const place = () => {{ + if (callout === null || callout.box !== box) return; + if (el.isConnected) {{ + const r = el.getBoundingClientRect(); + box.style.left = (r.left - pad) + 'px'; + box.style.top = (r.top - pad) + 'px'; + box.style.width = (r.width + 2 * pad) + 'px'; + box.style.height = (r.height + 2 * pad) + 'px'; + if (tab) {{ + // On the box's top left, outside it; below the box when the box is at the top. + const high = tab.offsetHeight; + const above = r.top - pad - high - 2 >= 0; + tab.style.left = Math.max(0, r.left - pad) + 'px'; + tab.style.top = (above ? r.top - pad - high - 2 : r.bottom + pad + 2) + 'px'; + }} + }} + callout.frame = requestAnimationFrame(place); + }}; + callout = {{ box, tab, frame: 0 }}; + place(); + const drawn = box.getBoundingClientRect(); + return {{ x: drawn.left, y: drawn.top, w: drawn.width, h: drawn.height }}; + }}; + window.__sceneCalloutClear = () => {{ + if (!callout) return; + cancelAnimationFrame(callout.frame); + callout.box.remove(); + if (callout.tab) callout.tab.remove(); + callout = null; }}; window.__sceneSay = (text) => {{ try {{ sessionStorage.setItem(KEY, text); }} catch (e) {{ /* a caption is not worth a failed take */ }} @@ -552,8 +720,13 @@ class BrowserStage: """The page being filmed, and the clock every mark is measured against.""" def __init__(self, page, out: Path, started: float, presses: list | None = None, - base: str = "", slides: dict | None = None, render=None, run_fsmes=None) -> None: + base: str = "", slides: dict | None = None, render=None, run_fsmes=None, + type_delay: float = TYPE_DELAY_SECONDS) -> None: self.page = page + #: Seconds between typed keys: the scene's `type_delay_seconds`. + self.type_delay = type_delay + #: Which step is being played, counted from 1 as the scene file's errors count them. + self.step_number = 0 #: Runs `fsmes ` against the take's own plant and returns what it #: printed; a terminal card's command goes through it. None without a plant. self.run_fsmes = run_fsmes @@ -581,6 +754,7 @@ def _at(self) -> float: def begin(self, step: dict) -> None: """The step about to be played: its part goes on every mark it makes.""" self.part = step.get("part") + self.step_number += 1 def note(self, what: str, kind: str, at: float | None = None, **rest: object) -> None: mark = {"at": self._at() if at is None else at, "what": what, "kind": kind, @@ -629,11 +803,14 @@ def click(self, selector: str) -> None: # takes itself away, a link that leaves the page) the press is all # there is, and the mark says so. self.clear_of_caption(selector) + # A click's own `x`, `y` is where the thing settled, measured below. + pointed = {key: value for key, value in self.point_at(self.page.locator(selector).first).items() + if key not in ("x", "y")} heard = len(self.presses) # The same element `page.click` will press: the first that matches. clicked = self.page.locator(selector).first.element_handle() self.page.click(selector) - self.note(f"click {selector}", "click", x=None, y=None) + self.note(f"click {selector}", "click", x=None, y=None, **pointed) mark = self.marks[-1] begun = time.monotonic() settled = settled_centre(self.page, clicked, selector) @@ -644,6 +821,10 @@ def click(self, selector: str) -> None: self.page.wait_for_timeout(int(rest * 1000)) press = self.presses[heard] if len(self.presses) > heard else None mark.update(click_position(press, settled)) + if mark["x"] is not None: + # The frames after the click show the thing where the page settled + # it, and the cursor is drawn on it there. + self.cursor_to(mark["x"], mark["y"], CURSOR_FOLLOW_SECONDS) if mark["x"] is None: mark["what"] += " (where it landed was neither heard from the page nor measured, so it is not given)" print(f" at {mark['x']},{mark['y']} press {mark.get('press')}" @@ -654,20 +835,58 @@ def fill(self, selector: str, text: str) -> None: # words arriving rather than a field that was blank and then full. self.clear_of_caption(selector) field = self.page.locator(selector).first + pointed = self.point_at(field) + typing = time.monotonic() field.fill("") if text: - field.press_sequentially(text, delay=int(TYPE_DELAY_SECONDS * 1000)) - self.note(f"fill {selector}: {text!r}", "fill", text=text) + # Rounded up: a delay of 84 ms, not 85 truncated to 84 and then some. + field.press_sequentially(text, delay=math.ceil(self.type_delay * 1000)) + typed = round(time.monotonic() - typing, 2) + self.note(f"fill {selector}: {text!r}", "fill", text=text, typed_seconds=typed, **pointed) self.page.wait_for_timeout(int(SMALL_STEP_SECONDS * 1000)) def choose(self, selector: str, value: str) -> None: # The option is picked, not the list opened: an open ``), `value` | | +| `wait_for` | `selector` with `text` / `attribute`+`equals` / `js`, or `js` | `timeout` | +| `say` | `line` | `hold`, `at`, `label`, `style` | +| `hold` | | `seconds` | +| `mark` | `name` | | +| `slide` | `name`, `title` | `lines`, `command`, `start_at`, `stop_after`, `hold` | + +Any step may also have `part` and `scroll_to` (with `scroll_block`, +`scroll_offset`, `scroll_text`). + +`click`, `fill` and `choose` move a drawn cursor to the thing and press it +with a ripple. A `` cannot be opened by a script, so on `choose` the cursor presses the list and the value changes. A `say` with `at` (a selector, as `click` takes) draws a box round that thing -as the caption appears, for the line's hold; `label` puts a few words on a tab -at the box's top left (below it when the box is at the top of the frame); -`style = "spotlight"` dims the rest of the frame instead. A thing off the -frame is scrolled onto it first; a thing not on the page refuses the take, -naming the step. `label` or `style` without `at` is refused. +as the caption appears, for the line's hold, and through any `mark` straight +after it, so those stills show the box; `label` puts a few words on a tab +at the box's top left (below it when the box is at the top of the frame, +inside it when the box fills the frame); `style = "spotlight"` dims the rest +of the frame instead. A thing off the frame, or under the page's sticky +header or the caption bar, is scrolled clear first; a thing not on the page +refuses the take, naming the step. Point `at` something smaller than the +frame. `label` or `style` without `at` is refused. diff --git a/tests/test_the_recorder_draws_a_cursor_and_a_box_a_viewer_can_follow.py b/tests/test_the_recorder_draws_a_cursor_and_a_box_a_viewer_can_follow.py index ec34e7a5..14ce21f5 100644 --- a/tests/test_the_recorder_draws_a_cursor_and_a_box_a_viewer_can_follow.py +++ b/tests/test_the_recorder_draws_a_cursor_and_a_box_a_viewer_can_follow.py @@ -209,15 +209,48 @@ def test_a_say_pointing_at_something_not_on_the_page_refuses_the_take_with_its_s assert stage.page.locator("#fs-scene-callout").count() == 0, "a box was drawn for nothing" -def test_the_box_is_gone_before_the_next_step(stage): +def test_the_box_stays_through_a_mark_after_its_line_and_is_gone_before_the_next_step(stage): stage, base = stage stage.goto(f"{base}/dashboard", 0.1) stage.say("Plant OEE.", 0.1, at="#kpi-oee", label="Plant OEE") + stage.begin({"do": "mark"}) + assert stage.page.locator("#fs-scene-callout").count() == 1, "a still taken after the line has no box" + + stage.begin({"do": "click"}) assert stage.page.locator("#fs-scene-callout").count() == 0 assert stage.page.locator("#fs-scene-callout-label").count() == 0 +#: Is the point at the middle of the element's top band its own, or does +#: something drawn over it (the sticky header) answer instead? +SEEN = """(sel) => { + const el = document.querySelector(sel); + const r = el.getBoundingClientRect(); + const x = r.left + r.width / 2; + return [r.top + 2, r.top + r.height / 2].map((y) => el.contains(document.elementFromPoint(x, y))); +}""" + + +def test_a_say_at_a_thing_under_the_sticky_header_scrolls_it_out_and_its_label_clear(stage): + stage, base = stage + stage.goto(f"{base}/dashboard", 0.1) + page = stage.page + page.evaluate("""() => { + const r = document.querySelector('#m-q').getBoundingClientRect(); + window.scrollBy({top: r.top - 20, behavior: 'instant'}); + }""") + assert page.evaluate(SEEN, "#m-q") == [False, False], "the field is not under the header to begin with" + + stage.say("Type part of a machine's name.", 0.1, at="#m-q", label="Filter") + + assert page.evaluate(SEEN, "#m-q") == [True, True], "the boxed field is still under the header" + header = page.evaluate("() => document.querySelector('header').getBoundingClientRect().bottom") + tab = page.evaluate("() => document.getElementById('fs-scene-callout-label').getBoundingClientRect().top") + assert tab >= header, f"the label's top {tab} is on the header, which ends at {header}" + assert stage.marks[-1]["box"]["y"] >= header, stage.marks[-1] + + def test_the_label_of_a_box_that_fills_the_frame_is_drawn_inside_it_where_it_can_be_read(stage): stage, base = stage stage.goto(f"{base}/dashboard", 0.1)