From f1e22bad1a24ca5cceac0cccf948255bf5b2d9de Mon Sep 17 00:00:00 2001 From: PMullz Date: Fri, 18 Sep 2026 21:29:46 -0400 Subject: [PATCH 1/5] docs(plans): voice input and text-to-speech design Design for #54. Records the researched platform constraints that shape the feature, most importantly that Web Speech recognition is server-based by default, so the issue's on-device privacy claim cannot be documented as written. Refs #54 Co-Authored-By: Claude Fable 5.1 --- .../2026-09-18-voice-input-tts-design.md | 307 ++++++++++++++++++ 1 file changed, 307 insertions(+) create mode 100644 docs/plans/2026-09-18-voice-input-tts-design.md diff --git a/docs/plans/2026-09-18-voice-input-tts-design.md b/docs/plans/2026-09-18-voice-input-tts-design.md new file mode 100644 index 0000000..dbb4bbc --- /dev/null +++ b/docs/plans/2026-09-18-voice-input-tts-design.md @@ -0,0 +1,307 @@ +# Voice input and text-to-speech playback: design + +Issue: [#54](https://github.com/PMDevSolutions/Claudius/issues/54) +Date: 2026-09-18 +Status: implemented in the same PR as this document + +## Summary + +Add an opt-in `voice` option to the widget. It puts a mic button in the +composer that dictates into the message field through the browser's +`SpeechRecognition`, and a read-aloud control under each completed assistant +message that plays it through `speechSynthesis`. Both halves are feature +detected independently and render nothing where the browser lacks them. + +The feature is widget-only. No worker, API, or wire-format change. + +## Facts that constrain the design + +These were checked against MDN and the MDN browser-compat data on 2026-09-18, +because several of them contradict common assumptions (including one in the +issue). + +1. **Recognition audio usually leaves the device.** MDN: "By default, using + speech recognition on a web page involves a server-based recognition + engine. Your audio is sent to a web service for recognition processing." + Chrome sends audio to Google, Safari hands it to Apple's dictation service. + The issue's acceptance criterion ("speech is processed on-device by the + browser; no audio leaves the client") is therefore **not true** and must + not be documented as written. What *is* true, and what we document: the + widget never requests microphone access itself, never sees audio, and + never sends audio to the Claudius worker. It receives text from the + browser, exactly as if it had been typed. +2. **On-device recognition exists but is not ready to build on.** + `SpeechRecognition.processLocally` shipped in desktop Chrome 139. It is + marked experimental, is unsupported on Chrome for Android, Safari, and + Firefox, needs a language-pack download flow (`available()` / + `install()`), and has open bugs where `available()` hangs. Out of scope + here; listed under follow-ups. +3. **The API exposes no audio signal.** There is no level, waveform, or + sample access. A true amplitude meter needs a second microphone capture + through `getUserMedia` plus a Web Audio `AnalyserNode`. +4. **Support matrix.** Recognition: Chrome and Edge 33+ (prefixed; unprefixed + from 139), Safari 14.1+ and iOS Safari 14.5+ (prefixed), Firefox only + behind a flag. Synthesis: every current browser except Android WebView. + Some Chromium forks (Brave, some in-app browsers) expose the constructor + but fail at `start()` with a `network` or `service-not-allowed` error, so + detection alone is not enough: start-time errors need a visible message. +5. **`speechSynthesis.pause()` is `cancel()` on Android.** Compat data: "In + Android, `pause()` ends the current utterance." +6. **Chrome cuts long utterances off** at roughly 15 seconds when a network + voice is used, and then fires no `end` event. +7. **Network voices send the text being read to the voice vendor.** + `SpeechSynthesisVoice.localService` distinguishes them. + +## Approaches considered + +**A. Two small hooks over the browser APIs, no dependencies (chosen).** +`useSpeechRecognition` and `useSpeechSynthesis` wrap the platform, pure +helpers hold the logic worth unit testing, and the UI stays thin. Smallest +bundle cost, no global state, works with multiple widget instances. + +**B. A library** (`react-speech-recognition`, `react-speech-kit`). Rejected: +more bytes than writing it directly, singleton state that fights multiple +instances, and nothing for the Android pause or 15-second problems. + +**C. Server-side STT/TTS through the worker.** Would cover Firefox, but sends +audio to our worker, which is the opposite of the privacy goal, and adds +cost, latency, and a large worker surface. Rejected for this issue. + +## Public API + +```ts +type VoiceInputMode = "toggle" | "hold"; + +interface VoiceOptions { + input?: boolean; // mic button. Default true + output?: boolean; // read-aloud control. Default true + mode?: VoiceInputMode; // Default "toggle" + autoSubmit?: boolean; // send when dictation ends. Default false + lang?: string; // BCP-47. Default: derived from the widget locale +} + +// ChatWidgetProps / window.ClaudiusConfig +voice?: boolean | VoiceOptions; // Default false +``` + +- **Off by default**, like `attachments`. A minor release must not add a + microphone button to every existing embed, and turning it on has privacy + implications the site owner should choose deliberately. +- `voice: true` enables both halves. `output: false` makes the speaker + optional, as the issue asks. `input: false` gives a read-aloud-only widget. + Both `false` resolves to disabled. +- An unknown `mode` falls back to `"toggle"` rather than throwing, since it + can arrive from an HTML attribute. +- `` gains `voice`, `voice-mode`, `voice-auto-submit`, + `voice-input`, and `voice-output`. Unlike attachment limits these are plain + enums and booleans, and the web component cannot read `ClaudiusConfig`, so + without them "configurable per widget" would not hold for that embed style. +- `clients/_schema.json`, CLI validation, and both snippet generators accept + `widget.voice`. + +### Language + +`resolveSpeechLang(locale, override)` returns, in order: + +1. `voice.lang` when set. +2. `` when it carries a region and its primary subtag matches the + widget locale (`en-GB` on an English widget). +3. `navigator.language` under the same rule. +4. A default per built-in locale: `en-US`, `es-ES`, `fr-FR`, `de-DE`. + +Steps 2 and 3 mirror `detectLocale()` and mean a UK site gets British +recognition and a British voice with no configuration. With nothing +detectable the result is `en-US`, as the issue specifies. + +## Architecture + +``` +ChatWidget resolveVoiceConfig(voice, locale ?? detectLocale()) + ChatWindow useSpeechSynthesis(lang) one reader per window + ChatMessage completed assistant messages + ChatInput useSpeechRecognition(lang) + +``` + +| Unit | Responsibility | +|------|----------------| +| `utils/voice.ts` | `resolveVoiceConfig`, `resolveSpeechLang`, `joinDictation`, `chunkSpeechText`, `pickVoice`. Pure. | +| `hooks/useSpeechRecognition.ts` | One recognition session at a time: start, stop, abort, transcript, error mapping. Minimal local typings, since TypeScript's DOM lib has none for recognition. | +| `hooks/useSpeechSynthesis.ts` | Reads one message at a time: chunk queue, pause, resume, cancel. | +| `components/VoiceInputButton.tsx` | The mic button and its toggle / hold gesture handling. Presentational. | +| `components/VoiceLevel.tsx` | The listening indicator. Presentational. | +| `components/MessageSpeechControls.tsx` | Read aloud / pause / resume / stop buttons. Presentational. | + +`ChatInput` owns the recognition hook because the transcript has to land in +its controlled field. `ChatWindow` owns the synthesis hook because only one +message may speak at a time and it already holds `streamingMessageId`. + +## Behavior + +### Dictation + +- A **new recognition instance per session**, created in `start()`. Reusing + one after an error is unreliable in Safari. +- `interimResults = true` so words appear in the field as they are spoken. + `continuous = false` in both modes. Continuous mode duplicates results on + Chrome for Android and misreports `isFinal` in Safari; a single utterance + per session is what a chat message needs. If the engine ends the session + during a pause, pressing the mic again appends. +- Dictation **appends** to whatever is already typed. The field value at + `start()` is the base; each result renders `joinDictation(base, transcript)` + clipped to the existing 2000 character limit. +- **Typing takes over.** A manual edit while listening keeps the edited text + and aborts the session, so a late result cannot overwrite it. +- `isListening` turns true at `start()`, not at the `start` event. That gives + immediate feedback and guards against a double start while the permission + prompt is open. +- **Toggle mode:** click starts, click stops. The session also ends by itself + after the utterance. +- **Hold mode:** pointer down starts and pointer up, cancel, or lost capture + stops; Space and Enter work the same way from the keyboard. Releasing + before the engine has actually started calls `abort()` rather than + `stop()`, so a permission prompt answered after release can never leave the + microphone open. A `click` with no preceding pointer or key press, which is + how screen readers and switch devices activate a button, toggles instead, + so hold mode stays usable without a sustained press. +- **Auto-submit** fires only when the session ended normally and produced + final text. Never after an error, an abort, or silence. It goes through the + same submit path as the send button, so pending attachments, the length + limit, and plugins all apply. +- Starting dictation **cancels any read-aloud** so the microphone does not + transcribe the widget's own voice. +- Unmounting (closing the chat) aborts the session. + +### Listening indicator + +Animated bars sit inside the right edge of the message field while listening, +and the placeholder reads "Listening...". The bars have two states, idle and +active, driven by the engine's `speechstart` / `speechend` events and by +arriving results. + +This is a voice-activity indicator, **not an amplitude meter**, and that is a +deliberate deviation from a literal reading of "waveform / level indicator". +A true meter needs the second `getUserMedia` capture described above, which +(a) conflicts with the recognizer's own capture on Android and iOS, the +platforms the issue cares most about, (b) adds a second permission prompt in +Safari, and (c) would give widget code access to raw audio, which would make +the privacy statement in the next section untrue. The indicator still +reflects whether speech is being heard, which is the feedback a user needs. + +Under `prefers-reduced-motion` the bars do not animate; the two states differ +in opacity. + +### Read aloud + +- Offered on assistant messages that have settled (not the streaming one) and + have text. Nothing plays automatically. +- Text passes through the existing `stripAnnouncementFormatting`, so markdown + markers are not read out and URLs collapse to hostnames. +- The text is split at sentence boundaries into chunks of at most 200 + characters and queued as separate utterances. That avoids Chrome's + 15-second cutoff and gives clean pause points. Utterances are held in a ref + because Safari garbage-collects them mid-speech otherwise. +- `pickVoice` prefers a **local** voice for the language, then any voice for + the language, else leaves the choice to the browser via `utterance.lang`. + Local voices keep the text on the device and do not have the cutoff bug. + Android reports `en_US` style tags, so matching normalizes the separator. +- Controls: idle shows Read aloud; speaking shows Pause and Stop; paused + shows Resume and Stop. Starting another message stops the current one. +- A session counter makes late events from cancelled utterances harmless. +- **Android pause:** if a chunk ends while a pause was requested and the + engine does not report `paused`, the platform treated pause as cancel. The + rest of the queue is cancelled and the control returns to idle, instead of + the next sentence starting to play after the user pressed pause. + +## Errors + +Recognition errors map to four translated messages shown in the composer's +existing `role="alert"` slot: + +| Engine code | Message | +|-------------|---------| +| `not-allowed`, `service-not-allowed` | Microphone access is blocked | +| `no-speech` | No speech was detected | +| `audio-capture` | No microphone was found | +| `network`, `language-not-supported`, anything else, `start()` throwing | Voice input is unavailable | +| `aborted` | ignored | + +The message clears on the next start or edit. Synthesis errors return the +control to idle silently; there is nothing useful to tell the user. + +## Accessibility + +- The mic is a real `button` with a constant label and `aria-pressed`, rather + than a label that changes with state, to avoid announcing the state twice. +- No live "listening" announcement: a screen reader speaking at that moment + would be transcribed. +- Indicator bars are `aria-hidden`. State is carried by `aria-pressed` and + the placeholder. +- Composer button is 44 px like its neighbours; message controls are 28 px, + above the WCAG 2.5.8 minimum. +- Hold mode has the click fallback described above (WCAG 2.5.1). +- New buttons are picked up by the existing focus trap, which queries + focusable elements on each Tab. + +## Strings + +Eleven keys, translated in en, es, fr, and de (the parity test enforces it): +`voiceInput`, `voiceInputHold`, `voiceListening`, `voicePermissionDenied`, +`voiceNoSpeech`, `voiceNoMicrophone`, `voiceUnavailable`, `readAloud`, +`pauseReading`, `resumeReading`, `stopReading`. + +## Testing + +Test-first, with fake `SpeechRecognition` and `speechSynthesis` +implementations that tests drive event by event. + +- `utils/voice`: config resolution, language resolution, joining, chunking, + voice picking. +- `useSpeechRecognition`: unsupported no-op, session setup, interim and final + transcripts, stop versus abort, each error mapping, `start()` throwing, + double start, unmount. +- `useSpeechSynthesis`: chunk queue with language and voice, state from + events, pause and resume, the Android pause path, switching messages, stale + events after cancel, unmount, unsupported no-op. +- Components: visibility rules, both modes by pointer, keyboard, and bare + click, appending, typing takes over, auto-submit and the cases where it + must not fire, error display, read-aloud only on settled assistant + messages, dictation cancels read-aloud. +- Embed attributes, client-config validation, snippets. +- One Playwright spec with injected fakes to prove the wiring in the built + app. Real engines cannot run in CI: no microphone, and Chromium builds ship + without the recognition service key. + +## Documentation + +New `configuration/voice.md`: enabling, options, browser support table, where +voices come from per browser and how to list them, and the privacy posture as +stated under fact 1. Updates to the widget and clients configuration pages, +the FAQ, and `CLAUDE.md`. + +## Bundle budget + +Everything ships in the main bundle: the IIFE build cannot code-split. +Budgets are raised to measured size plus 5%, in their own commit, per +CONTRIBUTING. + +## Acceptance criteria + +| Criterion | Status | +|-----------|--------| +| Mic visible only when recognition is supported | Met | +| Hold and toggle modes, configurable per widget | Met, across all three embed styles | +| Inline waveform / level indicator | Met as a voice-activity indicator, not an amplitude meter. See Listening indicator | +| Auto-submit, off by default | Met, as a configuration option | +| Speaker with cancel and pause | Met. Pause degrades to stop on Android, a platform limit | +| Locale follows the widget, `en-US` default; voices documented | Met | +| Documented privacy posture | Met, **with corrected wording**. The issue's claim is inaccurate. See fact 1 | +| Graceful no-op when unsupported | Met | + +## Out of scope, possible follow-ups + +- `onDevice` option built on `processLocally`, including the language-pack + install flow, once the API is stable beyond desktop Chrome. +- Opt-in amplitude meter for desktop browsers. +- Auto-reading new replies, voice / rate / pitch selection, read-aloud for + the welcome message. From f7233ae3100a9691d9e54b873b5d963ea8b3d44c Mon Sep 17 00:00:00 2001 From: PMullz Date: Fri, 18 Sep 2026 21:59:43 -0400 Subject: [PATCH 2/5] feat(widget): voice input and text-to-speech playback Add an opt-in `voice` option (React prop, `window.ClaudiusConfig` key, and `` attributes). Off by default, like attachments. Dictation: a mic button in the composer, shown only where the browser has SpeechRecognition. Toggle (default) or hold-to-talk, with interim results streaming into the message field and appending to typed text. Typing takes over from an open session, and sending ends it. Optional auto-submit fires only after a session that ended normally and heard something. A voice-activity indicator sits inside the field while listening. Read aloud: a speaker control under settled assistant replies, with pause, resume, and stop. One reply at a time, silenced when dictation starts or the chat closes. Markdown is stripped the same way as the screen-reader announcement. Platform quirks handled, each pinned by a test: - stop() before the engine has started aborts instead, so answering the permission prompt after release cannot open the mic - replies are queued as sentence-sized utterances to dodge Chrome's ~15 s cutoff - on Android pause() ends the utterance; the rest of the queue is dropped rather than read on after a "pause" - late error events from cancelled utterances are ignored - an idle engine is not sent cancel() right before speak() - hold mode still toggles on a click with no click count, which is how screen readers and switch devices activate a button Speech language follows the widget locale, refined by the region in or the browser language (en-GB page -> British recognition), defaulting to en-US. Eleven new strings in en, es, fr, and de. The test setup gains a PointerEvent polyfill: jsdom has none, so Testing Library was dispatching bare Events with no `button` or `pointerId`. Refs #54 Co-Authored-By: Claude Fable 5.1 --- widget/e2e/voice.spec.ts | 194 ++++++++++ widget/src/__tests__/embed.test.tsx | 49 +++ widget/src/components/ChatInput.stories.tsx | 21 ++ widget/src/components/ChatInput.tsx | 137 ++++++- widget/src/components/ChatMessage.stories.tsx | 38 ++ widget/src/components/ChatMessage.tsx | 25 ++ widget/src/components/ChatWidget.tsx | 22 +- widget/src/components/ChatWindow.tsx | 46 +++ .../src/components/MessageSpeechControls.tsx | 88 +++++ widget/src/components/VoiceInputButton.tsx | 141 +++++++ widget/src/components/VoiceLevel.tsx | 25 ++ .../__tests__/ChatInput.voice.test.tsx | 345 ++++++++++++++++++ .../__tests__/ChatMessage.voice.test.tsx | 47 +++ .../__tests__/ChatWidget.voice.test.tsx | 77 ++++ .../__tests__/ChatWindow.voice.test.tsx | 231 ++++++++++++ .../__tests__/MessageSpeechControls.test.tsx | 61 ++++ .../__tests__/VoiceInputButton.test.tsx | 153 ++++++++ widget/src/embed.tsx | 10 + .../__tests__/useSpeechRecognition.test.ts | 325 +++++++++++++++++ .../__tests__/useSpeechSynthesis.test.ts | 157 ++++++++ widget/src/hooks/useSpeechRecognition.ts | 195 ++++++++++ widget/src/hooks/useSpeechSynthesis.ts | 115 ++++++ widget/src/i18n.ts | 22 ++ widget/src/index.ts | 5 + widget/src/locales/de.ts | 15 + widget/src/locales/en.ts | 14 + widget/src/locales/es.ts | 14 + widget/src/locales/fr.ts | 14 + widget/src/main.tsx | 2 +- widget/src/styles.css | 56 +++ widget/src/test-setup.ts | 18 + .../src/test-utils/fakeSpeechRecognition.ts | 115 ++++++ widget/src/test-utils/fakeSpeechSynthesis.ts | 133 +++++++ widget/src/utils/__tests__/voice.test.ts | 256 +++++++++++++ widget/src/utils/voice.ts | 219 +++++++++++ 35 files changed, 3363 insertions(+), 22 deletions(-) create mode 100644 widget/e2e/voice.spec.ts create mode 100644 widget/src/components/MessageSpeechControls.tsx create mode 100644 widget/src/components/VoiceInputButton.tsx create mode 100644 widget/src/components/VoiceLevel.tsx create mode 100644 widget/src/components/__tests__/ChatInput.voice.test.tsx create mode 100644 widget/src/components/__tests__/ChatMessage.voice.test.tsx create mode 100644 widget/src/components/__tests__/ChatWidget.voice.test.tsx create mode 100644 widget/src/components/__tests__/ChatWindow.voice.test.tsx create mode 100644 widget/src/components/__tests__/MessageSpeechControls.test.tsx create mode 100644 widget/src/components/__tests__/VoiceInputButton.test.tsx create mode 100644 widget/src/hooks/__tests__/useSpeechRecognition.test.ts create mode 100644 widget/src/hooks/__tests__/useSpeechSynthesis.test.ts create mode 100644 widget/src/hooks/useSpeechRecognition.ts create mode 100644 widget/src/hooks/useSpeechSynthesis.ts create mode 100644 widget/src/test-utils/fakeSpeechRecognition.ts create mode 100644 widget/src/test-utils/fakeSpeechSynthesis.ts create mode 100644 widget/src/utils/__tests__/voice.test.ts create mode 100644 widget/src/utils/voice.ts diff --git a/widget/e2e/voice.spec.ts b/widget/e2e/voice.spec.ts new file mode 100644 index 0000000..5cb723f --- /dev/null +++ b/widget/e2e/voice.spec.ts @@ -0,0 +1,194 @@ +import { test, expect, type Page } from "@playwright/test"; +import { mockChatApi } from "./helpers"; + +/** + * Voice features against fake speech engines, injected before the app loads. + * Real engines cannot run here: CI has no microphone, and Chromium builds ship + * without the key for Google's recognition service. The fakes let these specs + * prove the wiring in a real browser; engine behaviour is covered by the unit + * tests in src/hooks/__tests__. + */ +async function installFakeSpeech(page: Page) { + await page.addInitScript(() => { + type Handler = ((event: unknown) => void) | null; + + class FakeRecognition { + lang = ""; + continuous = false; + interimResults = false; + onstart: Handler = null; + onend: Handler = null; + onerror: Handler = null; + onresult: Handler = null; + onspeechstart: Handler = null; + onspeechend: Handler = null; + start() { + (window as unknown as { __recognition: FakeRecognition }).__recognition = + this; + this.onstart?.({}); + } + stop() { + this.onend?.({}); + } + abort() {} + hear(transcript: string, isFinal: boolean) { + this.onresult?.({ + results: [{ isFinal, length: 1, 0: { transcript } }], + }); + } + } + + const spoken: string[] = []; + const synth = { + speaking: false, + pending: false, + paused: false, + spoken, + speak(utterance: SpeechSynthesisUtterance) { + spoken.push(`${utterance.lang}|${utterance.text}`); + this.speaking = true; + }, + cancel() { + this.speaking = false; + this.paused = false; + }, + pause() { + this.paused = true; + }, + resume() { + this.paused = false; + }, + getVoices: () => [], + }; + + const w = window as unknown as Record; + // Chrome 139+ exposes the unprefixed constructor too, and the widget + // prefers it, so replace both or the real engine would be started. + w.SpeechRecognition = FakeRecognition; + w.webkitSpeechRecognition = FakeRecognition; + Object.defineProperty(window, "speechSynthesis", { + value: synth, + configurable: true, + }); + }); +} + +function hear(page: Page, transcript: string, isFinal: boolean) { + return page.evaluate( + ([text, final]) => + ( + window as unknown as { + __recognition: { hear(t: string, f: boolean): void }; + } + ).__recognition.hear(text as string, final as boolean), + [transcript, isFinal] as const, + ); +} + +test.describe("voice input and read-aloud", () => { + test("dictates into the message field, then sends like a typed message", async ({ + page, + }) => { + await installFakeSpeech(page); + const api = await mockChatApi(page); + api.enqueueReply("Plans start at $10."); + await page.goto("/"); + await page.getByRole("button", { name: /open chat/i }).click(); + + const mic = page.getByRole("button", { name: "Voice input" }); + const input = page.getByLabel(/type your message/i); + await mic.click(); + await expect(mic).toHaveAttribute("aria-pressed", "true"); + await expect(input).toHaveAttribute("placeholder", "Listening..."); + + await hear(page, "what are your pri", false); + await expect(input).toHaveValue("what are your pri"); + await hear(page, "what are your prices", true); + + // Clicking again stops; the text stays for review rather than auto-sending. + await mic.click(); + await expect(mic).toHaveAttribute("aria-pressed", "false"); + await expect(input).toHaveValue("what are your prices"); + expect(api.callCount()).toBe(0); + + await input.press("Enter"); + const log = page.getByRole("log"); + await expect(log.getByText("what are your prices")).toBeVisible(); + await expect(log.getByText("Plans start at $10.")).toBeVisible(); + }); + + test("keeps the newest words in view during a long dictation", async ({ + page, + }) => { + await installFakeSpeech(page); + await page.goto("/"); + await page.getByRole("button", { name: /open chat/i }).click(); + await page.getByRole("button", { name: "Voice input" }).click(); + + await hear(page, "and then some more words ".repeat(10), false); + + // The field is not focused while dictating, so the browser will not + // scroll it by itself; without help it would show only the first words. + const input = page.getByLabel(/type your message/i); + await expect + .poll(() => input.evaluate((el: HTMLInputElement) => el.scrollLeft)) + .toBeGreaterThan(0); + }); + + test("reads a reply aloud with pause and stop controls", async ({ page }) => { + await installFakeSpeech(page); + const api = await mockChatApi(page); + api.enqueueReply("Plans start at **$10** a month."); + await page.goto("/"); + await page.getByRole("button", { name: /open chat/i }).click(); + + const input = page.getByLabel(/type your message/i); + await input.fill("Prices?"); + await input.press("Enter"); + + const log = page.getByRole("log"); + await log.getByRole("button", { name: "Read aloud" }).click(); + + // Markdown markers are stripped before the text reaches the engine. + const spoken = await page.evaluate( + () => + (window as unknown as { speechSynthesis: { spoken: string[] } }) + .speechSynthesis.spoken, + ); + expect(spoken).toEqual(["en-US|Plans start at $10 a month."]); + + await log.getByRole("button", { name: "Pause reading" }).click(); + await expect( + log.getByRole("button", { name: "Resume reading" }), + ).toBeVisible(); + + await log.getByRole("button", { name: "Stop reading" }).click(); + await expect(log.getByRole("button", { name: "Read aloud" })).toBeVisible(); + }); + + test("shows no voice controls in a browser without the Web Speech API", async ({ + page, + }) => { + await page.addInitScript(() => { + const w = window as unknown as Record; + delete w.webkitSpeechRecognition; + delete w.SpeechRecognition; + Object.defineProperty(window, "speechSynthesis", { + value: undefined, + configurable: true, + }); + }); + const api = await mockChatApi(page); + api.enqueueReply("Plans start at $10."); + await page.goto("/"); + await page.getByRole("button", { name: /open chat/i }).click(); + + const input = page.getByLabel(/type your message/i); + await input.fill("Prices?"); + await input.press("Enter"); + await expect(page.getByRole("log").getByText("Plans start at $10.")).toBeVisible(); + + await expect(page.getByRole("button", { name: "Voice input" })).toHaveCount(0); + await expect(page.getByRole("button", { name: "Read aloud" })).toHaveCount(0); + }); +}); diff --git a/widget/src/__tests__/embed.test.tsx b/widget/src/__tests__/embed.test.tsx index 7e733a3..2ffd848 100644 --- a/widget/src/__tests__/embed.test.tsx +++ b/widget/src/__tests__/embed.test.tsx @@ -77,3 +77,52 @@ describe("embed attachments option", () => { ).toBeInTheDocument(); }); }); + +describe("embed voice option", () => { + beforeEach(() => { + vi.resetModules(); + document.body.innerHTML = ""; + window.sessionStorage.clear(); + window.ClaudiusConfig = undefined; + document.documentElement.lang = "en"; + (window as unknown as Record).webkitSpeechRecognition = + class { + start() {} + stop() {} + abort() {} + }; + }); + + afterEach(() => { + document.body.innerHTML = ""; + window.ClaudiusConfig = undefined; + document.documentElement.lang = ""; + delete (window as unknown as Record) + .webkitSpeechRecognition; + }); + + it("enables the mic from ClaudiusConfig", async () => { + window.ClaudiusConfig = { + apiUrl: "https://test.example/api", + voice: { mode: "hold" }, + }; + await import("../embed"); + (await screen.findByRole("button", { name: /open chat/i })).click(); + expect( + await screen.findByRole("button", { name: "Hold to talk" }), + ).toBeInTheDocument(); + }); + + it("enables the mic via web component attributes", async () => { + await import("../embed"); + const el = document.createElement("claudius-chat"); + el.setAttribute("api-url", "https://test.example/api"); + el.setAttribute("voice", ""); + el.setAttribute("voice-mode", "hold"); + document.body.appendChild(el); + (await screen.findByRole("button", { name: /open chat/i })).click(); + expect( + await screen.findByRole("button", { name: "Hold to talk" }), + ).toBeInTheDocument(); + }); +}); diff --git a/widget/src/components/ChatInput.stories.tsx b/widget/src/components/ChatInput.stories.tsx index db8dcdc..c969b8e 100644 --- a/widget/src/components/ChatInput.stories.tsx +++ b/widget/src/components/ChatInput.stories.tsx @@ -3,6 +3,15 @@ import { fn, userEvent, within } from "storybook/test"; import { ChatInput } from "./ChatInput"; import { locales, type LocaleCode } from "../locales"; import { DEFAULT_ATTACHMENT_OPTIONS } from "../utils/attachments"; +import type { ResolvedVoiceConfig } from "../utils/voice"; + +const VOICE: ResolvedVoiceConfig = { + input: true, + output: true, + mode: "toggle", + autoSubmit: false, + lang: "en-US", +}; const meta = { title: "Widget/ChatInput", @@ -78,3 +87,15 @@ export const WithPendingAttachments: Story = { await userEvent.upload(picker, [png, pdf]); }, }; + +// Mic button for dictation. It renders only in browsers with speech +// recognition (Chrome, Edge, Safari); click it, allow the microphone, and +// speak to see the listening indicator and the live transcript. +export const WithVoiceInput: Story = { + args: { voice: VOICE }, +}; + +// Hold-to-talk: records only while the button (or Space / Enter) is held. +export const WithVoiceHoldToTalk: Story = { + args: { voice: { ...VOICE, mode: "hold" } }, +}; diff --git a/widget/src/components/ChatInput.tsx b/widget/src/components/ChatInput.tsx index da12693..38f2976 100644 --- a/widget/src/components/ChatInput.tsx +++ b/widget/src/components/ChatInput.tsx @@ -11,6 +11,8 @@ import { import { defaultTranslations, type ClaudiusTranslations } from "../i18n"; import type { ChatAttachment } from "../api/types"; import { AttachmentPreview } from "./AttachmentPreview"; +import { VoiceInputButton } from "./VoiceInputButton"; +import { VoiceLevel } from "./VoiceLevel"; import { fileToAttachment, formatBytes, @@ -19,6 +21,11 @@ import { type ResolvedAttachmentsConfig, } from "../utils/attachments"; import { interpolate } from "../utils/interpolate"; +import { joinDictation, type ResolvedVoiceConfig } from "../utils/voice"; +import { + useSpeechRecognition, + type SpeechRecognitionErrorKind, +} from "../hooks/useSpeechRecognition"; const MAX_MESSAGE_LENGTH = 2000; const WARNING_THRESHOLD = 1800; @@ -37,6 +44,13 @@ interface ChatInputProps { * pasted or dropped files are ignored. */ attachments?: ResolvedAttachmentsConfig | null; + /** + * Voice settings. The mic button renders only when `voice.input` is on and + * the browser supports speech recognition. + */ + voice?: ResolvedVoiceConfig | null; + /** Called when dictation starts, so the parent can silence read-aloud. */ + onVoiceStart?: () => void; } function dragHasFiles(e: DragEvent): boolean { @@ -52,16 +66,28 @@ export function ChatInput({ placeholder, translations, attachments = null, + voice = null, + onVoiceStart, }: ChatInputProps) { const t = translations ?? defaultTranslations; const [value, setValue] = useState(""); const [pending, setPending] = useState([]); const [attachmentError, setAttachmentError] = useState(null); const [isDragOver, setIsDragOver] = useState(false); + const [voiceError, setVoiceError] = useState(null); const inputRef = useRef(null); const fileInputRef = useRef(null); const pendingRef = useRef([]); const dragDepthRef = useRef(0); + // Mirrors `value` for the speech callbacks, which fire outside a render. + const valueRef = useRef(""); + // What the field held when dictation began; speech is appended to it. + const dictationBaseRef = useRef(""); + + const updateValue = (next: string) => { + valueRef.current = next; + setValue(next); + }; const charCount = value.length; const isNearLimit = charCount >= WARNING_THRESHOLD; @@ -123,26 +149,82 @@ export function ChatInput({ setAttachmentError(null); }, []); - const handleSubmit = (e: FormEvent) => { - e.preventDefault(); - const trimmed = value.trim(); + const submit = () => { + const trimmed = valueRef.current.trim(); const files = pendingRef.current; - if ((!trimmed && files.length === 0) || isAtLimit) return; + if ( + (!trimmed && files.length === 0) || + valueRef.current.length >= MAX_MESSAGE_LENGTH + ) { + return; + } if (files.length > 0) { onSend(trimmed, files); } else { onSend(trimmed); } - setValue(""); + updateValue(""); pendingRef.current = []; setPending([]); setAttachmentError(null); }; + const describeVoiceError = (kind: SpeechRecognitionErrorKind): string => { + switch (kind) { + case "permission": + return t.voicePermissionDenied; + case "no-speech": + return t.voiceNoSpeech; + case "no-microphone": + return t.voiceNoMicrophone; + case "unavailable": + return t.voiceUnavailable; + } + }; + + const recognition = useSpeechRecognition({ + lang: voice?.lang ?? "en-US", + onTranscript: (transcript) => + updateValue( + joinDictation(dictationBaseRef.current, transcript, MAX_MESSAGE_LENGTH), + ), + onEnd: (transcript) => { + // Only a session that ended normally and actually heard something. + if (voice?.autoSubmit && transcript.trim()) submit(); + }, + onError: (kind) => setVoiceError(describeVoiceError(kind)), + }); + const showMic = !!voice?.input && recognition.isSupported; + const isListening = recognition.isListening; + + // The field is not focused while dictating, so the browser does not scroll + // it; without this only the first words of a long dictation are visible. + useEffect(() => { + const input = inputRef.current; + if (isListening && input) input.scrollLeft = input.scrollWidth; + }, [isListening, value]); + + const startDictation = () => { + setVoiceError(null); + dictationBaseRef.current = valueRef.current; + onVoiceStart?.(); + recognition.start(); + }; + + const handleSubmit = (e: FormEvent) => { + e.preventDefault(); + // Otherwise the next result would put the sent text back in the field. + recognition.abort(); + submit(); + }; + const handleChange = (e: ChangeEvent) => { const newValue = e.target.value; if (newValue.length <= MAX_MESSAGE_LENGTH) { - setValue(newValue); + // Typing takes over from dictation: keep the edit, drop the session. + recognition.abort(); + setVoiceError(null); + updateValue(newValue); } }; @@ -263,18 +345,33 @@ export function ChatInput({ )} - +
+ + {isListening && } +
+ {showMic && voice && ( + + )} {showStop ? ( + ); +} + +/** + * Read-aloud controls for an assistant message: a speaker button while idle, + * pause and stop while speaking, resume and stop while paused. + */ +export function MessageSpeechControls({ + state, + labels, + onPlay, + onPause, + onResume, + onStop, +}: MessageSpeechControlsProps) { + if (state === "idle") { + return ( + + + + + + ); + } + + return ( +
+ {state === "paused" ? ( + + + + ) : ( + + + + + )} + + + +
+ ); +} diff --git a/widget/src/components/VoiceInputButton.tsx b/widget/src/components/VoiceInputButton.tsx new file mode 100644 index 0000000..15eb660 --- /dev/null +++ b/widget/src/components/VoiceInputButton.tsx @@ -0,0 +1,141 @@ +import { + useRef, + type KeyboardEvent, + type MouseEvent, + type PointerEvent, +} from "react"; +import type { VoiceInputMode } from "../utils/voice"; + +interface VoiceInputButtonProps { + mode: VoiceInputMode; + isListening: boolean; + disabled?: boolean; + onStart: () => void; + onStop: () => void; + /** + * Accessible name. It stays the same in both states; `aria-pressed` carries + * the state, so screen readers don't announce it twice. + */ + label: string; +} + +function isHoldKey(e: KeyboardEvent): boolean { + return e.key === " " || e.key === "Enter"; +} + +/** + * Mic button for dictation. In `"toggle"` mode a click starts and the next + * click stops. In `"hold"` mode it records while pressed, by pointer or by + * Space / Enter. + */ +export function VoiceInputButton({ + mode, + isListening, + disabled = false, + onStart, + onStop, + label, +}: VoiceInputButtonProps) { + const isHold = mode === "hold"; + const holdingRef = useRef(false); + + const beginHold = () => { + if (disabled || holdingRef.current) return; + holdingRef.current = true; + onStart(); + }; + + // Idempotent: a release is often reported twice (pointerup, then + // lostpointercapture). + const endHold = () => { + if (!holdingRef.current) return; + holdingRef.current = false; + onStop(); + }; + + const handleClick = (e: MouseEvent) => { + // In hold mode the press handlers own pointer clicks. What is left is a + // click with no click count: a screen reader or switch device activating + // the button, which cannot hold it down. Let those toggle. + if (isHold && e.detail !== 0) return; + if (isListening) onStop(); + else onStart(); + }; + + const handlePointerDown = (e: PointerEvent) => { + if (!isHold || e.button !== 0) return; + // Keep receiving the release even if the finger slides off the button. + e.currentTarget.setPointerCapture?.(e.pointerId); + beginHold(); + }; + + const handleKeyDown = (e: KeyboardEvent) => { + if (!isHold || !isHoldKey(e)) return; + // Stop the browser turning the key press into a click as well. + e.preventDefault(); + if (!e.repeat) beginHold(); + }; + + const handleKeyUp = (e: KeyboardEvent) => { + if (!isHold || !isHoldKey(e)) return; + e.preventDefault(); + endHold(); + }; + + return ( + + ); +} diff --git a/widget/src/components/VoiceLevel.tsx b/widget/src/components/VoiceLevel.tsx new file mode 100644 index 0000000..bc65b25 --- /dev/null +++ b/widget/src/components/VoiceLevel.tsx @@ -0,0 +1,25 @@ +interface VoiceLevelProps { + /** True while the recognizer reports hearing speech. */ + active: boolean; +} + +/** + * Listening indicator shown inside the message field while dictating: bars + * that idle while the mic waits and liven up while speech is heard. It shows + * voice activity, not loudness. The Web Speech API exposes no audio signal, + * and metering one ourselves would mean opening the microphone a second time. + */ +export function VoiceLevel({ active }: VoiceLevelProps) { + return ( +